From 4d5b4c7863bed70c033fac431463054efc986b18 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 16 Sep 2021 20:02:54 +0200 Subject: [PATCH] Feature Extractor: Wav2Vec2 & Speech2Text - Allow truncation + padding=longest (#13600) * correct * add tests * Update src/transformers/feature_extraction_sequence_utils.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- .../feature_extraction_sequence_utils.py | 11 ++-- .../feature_extraction_speech_to_text.py | 2 +- .../wav2vec2/feature_extraction_wav2vec2.py | 2 +- .../test_feature_extraction_speech_to_text.py | 50 ++++++++++++++++++- tests/test_feature_extraction_wav2vec2.py | 36 ++++++++++++- 5 files changed, 91 insertions(+), 10 deletions(-) diff --git a/src/transformers/feature_extraction_sequence_utils.py b/src/transformers/feature_extraction_sequence_utils.py index 69a5511208..0ac97da7c2 100644 --- a/src/transformers/feature_extraction_sequence_utils.py +++ b/src/transformers/feature_extraction_sequence_utils.py @@ -211,16 +211,17 @@ class SequenceFeatureExtractor(FeatureExtractionMixin): for i in range(batch_size): inputs = dict((k, v[i]) for k, v in processed_features.items()) # truncation - inputs = self._truncate( + inputs_slice = self._truncate( inputs, max_length=max_length, pad_to_multiple_of=pad_to_multiple_of, truncation=truncation, ) - truncated_inputs.append(inputs) + truncated_inputs.append(inputs_slice) if padding_strategy == PaddingStrategy.LONGEST: - max_length = max(len(inputs) for inputs in required_input) + # make sure that `max_length` cannot be longer than the longest truncated length + max_length = max(len(input_slice[self.model_input_names[0]]) for input_slice in truncated_inputs) padding_strategy = PaddingStrategy.MAX_LENGTH batch_outputs = {} @@ -322,9 +323,7 @@ class SequenceFeatureExtractor(FeatureExtractionMixin): if not truncation: return processed_features elif truncation and max_length is None: - raise ValueError( - "When setting ``truncation=True``, make sure that ``max_length`` is defined and ``padding='max_length'``" - ) + raise ValueError("When setting ``truncation=True``, make sure that ``max_length`` is defined.") required_input = processed_features[self.model_input_names[0]] diff --git a/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py b/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py index da6394b0f1..d3c821b497 100644 --- a/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py +++ b/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py @@ -110,7 +110,7 @@ class Speech2TextFeatureExtractor(SequenceFeatureExtractor): std = np.sqrt(np.maximum(var, 1e-10)) x = np.divide(x, std) - if x.shape[0] > input_length: + if input_length < x.shape[0]: x[input_length:] = padding_value # make sure array is in float32 diff --git a/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py b/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py index c5fa52ecfb..6aa60df560 100644 --- a/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py +++ b/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py @@ -91,7 +91,7 @@ class Wav2Vec2FeatureExtractor(SequenceFeatureExtractor): for vector, length in zip(input_values, attention_mask.sum(-1)): normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7) - if length > normed_slice.shape[0]: + if length < normed_slice.shape[0]: normed_slice[length:] = padding_value normed_input_values.append(normed_slice) diff --git a/tests/test_feature_extraction_speech_to_text.py b/tests/test_feature_extraction_speech_to_text.py index 73522bcc07..e8df22ee28 100644 --- a/tests/test_feature_extraction_speech_to_text.py +++ b/tests/test_feature_extraction_speech_to_text.py @@ -189,10 +189,12 @@ class Speech2TextFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unitt self.assertTrue(np.all(np.abs(np.var(input_vector, axis=0) - 1) < var_tol)) _check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]], var_tol) + self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6) _check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]], var_tol) + self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6) _check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]], var_tol) - def test_cepstral_mean_and_variance_normalization_trunc(self): + def test_cepstral_mean_and_variance_normalization_trunc_max_length(self): feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()) speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] inputs = feature_extractor( @@ -214,3 +216,49 @@ class Speech2TextFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unitt _check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]) _check_zero_mean_unit_variance(input_features[1]) _check_zero_mean_unit_variance(input_features[2]) + + def test_cepstral_mean_and_variance_normalization_trunc_longest(self): + feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()) + speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] + inputs = feature_extractor( + speech_inputs, + padding="longest", + max_length=4, + truncation=True, + return_tensors="np", + return_attention_mask=True, + ) + input_features = inputs.input_features + attention_mask = inputs.attention_mask + fbank_feat_lengths = np.sum(attention_mask == 1, axis=1) + + def _check_zero_mean_unit_variance(input_vector): + self.assertTrue(np.all(np.mean(input_vector, axis=0) < 1e-3)) + self.assertTrue(np.all(np.abs(np.var(input_vector, axis=0) - 1) < 1e-3)) + + _check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]) + _check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]) + _check_zero_mean_unit_variance(input_features[2]) + + # make sure that if max_length < longest -> then pad to max_length + self.assertEqual(input_features.shape, (3, 4, 24)) + + speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] + inputs = feature_extractor( + speech_inputs, + padding="longest", + max_length=16, + truncation=True, + return_tensors="np", + return_attention_mask=True, + ) + input_features = inputs.input_features + attention_mask = inputs.attention_mask + fbank_feat_lengths = np.sum(attention_mask == 1, axis=1) + + _check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]) + _check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]) + _check_zero_mean_unit_variance(input_features[2]) + + # make sure that if max_length < longest -> then pad to max_length + self.assertEqual(input_features.shape, (3, 6, 24)) diff --git a/tests/test_feature_extraction_wav2vec2.py b/tests/test_feature_extraction_wav2vec2.py index 81c8d384fd..7165c364cb 100644 --- a/tests/test_feature_extraction_wav2vec2.py +++ b/tests/test_feature_extraction_wav2vec2.py @@ -135,7 +135,9 @@ class Wav2Vec2FeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest self.assertTrue(np.abs(np.var(input_vector) - 1) < 1e-3) _check_zero_mean_unit_variance(input_values[0][:800]) + self.assertTrue(input_values[0][800:].sum() < 1e-6) _check_zero_mean_unit_variance(input_values[1][:1000]) + self.assertTrue(input_values[0][1000:].sum() < 1e-6) _check_zero_mean_unit_variance(input_values[2][:1200]) def test_zero_mean_unit_variance_normalization(self): @@ -158,7 +160,7 @@ class Wav2Vec2FeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest _check_zero_mean_unit_variance(input_values[1][:1000]) _check_zero_mean_unit_variance(input_values[2][:1200]) - def test_zero_mean_unit_variance_normalization_trunc_np(self): + def test_zero_mean_unit_variance_normalization_trunc_np_max_length(self): feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()) speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] processed = feat_extract( @@ -174,6 +176,38 @@ class Wav2Vec2FeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest _check_zero_mean_unit_variance(input_values[1]) _check_zero_mean_unit_variance(input_values[2]) + def test_zero_mean_unit_variance_normalization_trunc_np_longest(self): + feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()) + speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] + processed = feat_extract( + speech_inputs, truncation=True, max_length=1000, padding="longest", return_tensors="np" + ) + input_values = processed.input_values + + def _check_zero_mean_unit_variance(input_vector): + self.assertTrue(np.abs(np.mean(input_vector)) < 1e-3) + self.assertTrue(np.abs(np.var(input_vector) - 1) < 1e-3) + + _check_zero_mean_unit_variance(input_values[0, :800]) + _check_zero_mean_unit_variance(input_values[1, :1000]) + _check_zero_mean_unit_variance(input_values[2]) + + # make sure that if max_length < longest -> then pad to max_length + self.assertTrue(input_values.shape == (3, 1000)) + + speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] + processed = feat_extract( + speech_inputs, truncation=True, max_length=2000, padding="longest", return_tensors="np" + ) + input_values = processed.input_values + + _check_zero_mean_unit_variance(input_values[0, :800]) + _check_zero_mean_unit_variance(input_values[1, :1000]) + _check_zero_mean_unit_variance(input_values[2]) + + # make sure that if max_length > longest -> then pad to longest + self.assertTrue(input_values.shape == (3, 1200)) + @slow @require_torch def test_pretrained_checkpoints_are_set_correctly(self):