SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add torchcodec in docstrings/tests for datasets 4.0 (#39156)

Browse Source 
* fix dataset run_object_detection

* bump version

* keep same dataset actually

* torchcodec in docstrings and testing utils

* torchcodec in dockerfiles and requirements

* remove duplicate

* add torchocodec to all the remaining docker files

* fix tests

* support torchcodec in audio classification and ASR

* [commit to revert] build ci-dev images

* [commit to revert] trigger circleci

* [commit to revert] build ci-dev images

* fix

* fix modeling_hubert

* backward compatible run_object_detection

* revert ci trigger commits

* fix mono conversion and support torch tensor as input

* revert map_to_array docs + fix it

* revert mono

* nit in docstring

* style

* fix modular

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Quentin Lhoest
2025-07-08 17:06:12 +02:00
committed by GitHub
parent 1255480fd2
commit 1ecd52e50a
78 changed files with 448 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.py
View File

@@ -154,7 +154,7 @@ class ASTFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.Test
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

2
tests/models/clap/test_feature_extraction_clap.py
View File

@@ -165,7 +165,7 @@ class ClapFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.Tes
def _load_datasamples(self, num_samples):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

2
tests/models/clvp/test_feature_extraction_clvp.py
View File

@@ -215,7 +215,7 @@ class ClvpFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.Tes
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
ds = ds.cast_column("audio", Audio(sampling_rate=22050))
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples], [x["sampling_rate"] for x in speech_samples]

9
tests/models/clvp/test_modeling_clvp.py
View File

@@ -373,10 +373,12 @@ class ClvpModelForConditionalGenerationTester:
ds = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
ds = ds.cast_column("audio", datasets.Audio(sampling_rate=22050))
_, audio, sr = ds.sort("id").select(range(1))[:1]["audio"][0].values()
audio = ds.sort("id")[0]["audio"]
audio_sample = audio["array"]
sr = audio["sampling_rate"]
feature_extractor = ClvpFeatureExtractor()
input_features = feature_extractor(raw_speech=audio, sampling_rate=sr, return_tensors="pt")[
input_features = feature_extractor(raw_speech=audio_sample, sampling_rate=sr, return_tensors="pt")[
"input_features"
].to(torch_device)
@@ -562,7 +564,8 @@ class ClvpIntegrationTest(unittest.TestCase):
self.text = "This is an example text."
ds = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
ds = ds.cast_column("audio", datasets.Audio(sampling_rate=22050))
_, self.speech_samples, self.sr = ds.sort("id").select(range(1))[:1]["audio"][0].values()
audio = ds.sort("id")["audio"][0]
self.speech_samples, self.sr = audio["array"], audio["sampling_rate"]
self.model = ClvpModelForConditionalGeneration.from_pretrained("susnato/clvp_dev").to(torch_device)
self.model.eval()

2
tests/models/dac/test_feature_extraction_dac.py
View File

@@ -143,7 +143,7 @@ class DacFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.Test
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
audio_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
audio_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in audio_samples]

4
tests/models/data2vec/test_modeling_data2vec_audio.py
View File

@@ -21,7 +21,7 @@ from datasets import load_dataset
from tests.test_modeling_common import floats_tensor, ids_tensor, random_attention_mask
from transformers import Data2VecAudioConfig, is_torch_available
from transformers.testing_utils import require_soundfile, require_torch, slow, torch_device
from transformers.testing_utils import require_torch, require_torchcodec, slow, torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init
@@ -656,7 +656,7 @@ class Data2VecAudioUtilsTest(unittest.TestCase):
@require_torch
@require_soundfile
@require_torchcodec
@slow
class Data2VecAudioModelIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):

2
tests/models/dia/test_feature_extraction_dia.py
View File

@@ -145,7 +145,7 @@ class DiaFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.Test
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
audio_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
audio_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in audio_samples]

8
tests/models/dia/test_modeling_dia.py
View File

@@ -665,8 +665,12 @@ class DiaForConditionalGenerationIntegrationTest(unittest.TestCase):
@require_torch_accelerator
def test_dia_model_integration_generate_audio_context(self):
text = ["[S1] Dia is an open weights text to dialogue model.", "This is a test"]
audio_sample_1 = torchaudio.load(self.audio_prompt_1_path, channels_first=True)[0].squeeze().numpy()
audio_sample_2 = torchaudio.load(self.audio_prompt_2_path, channels_first=True)[0].squeeze().numpy()
audio_sample_1 = (
torchaudio.load(self.audio_prompt_1_path, channels_first=True, backend="soundfile")[0].squeeze().numpy()
)
audio_sample_2 = (
torchaudio.load(self.audio_prompt_2_path, channels_first=True, backend="soundfile")[0].squeeze().numpy()
)
audio = [audio_sample_1, audio_sample_2]
processor = DiaProcessor.from_pretrained(self.model_checkpoint)

2
tests/models/encodec/test_feature_extraction_encodec.py
View File

@@ -139,7 +139,7 @@ class EnCodecFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
audio_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
audio_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in audio_samples]

2
tests/models/granite_speech/test_modeling_granite_speech.py
View File

@@ -340,7 +340,7 @@ class GraniteSpeechForConditionalGenerationIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

4
tests/models/hubert/test_modeling_hubert.py
View File

@@ -22,7 +22,7 @@ import unittest
import pytest
from transformers import HubertConfig, is_torch_available
from transformers.testing_utils import require_soundfile, require_torch, slow, torch_device
from transformers.testing_utils import require_torch, require_torchcodec, slow, torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
@@ -750,7 +750,7 @@ class HubertUtilsTest(unittest.TestCase):
@require_torch
@require_soundfile
@require_torchcodec
@slow
class HubertModelIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):

2
tests/models/kyutai_speech_to_text/test_modeling_kyutai_speech_to_text.py
View File

@@ -713,7 +713,7 @@ class KyutaiSpeechToTextForConditionalGenerationIntegrationTests(unittest.TestCa
def _load_datasamples(self, num_samples):
self._load_dataset()
ds = self._dataset
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]
@slow

2
tests/models/moonshine/test_modeling_moonshine.py
View File

@@ -443,7 +443,7 @@ class MoonshineModelIntegrationTests(unittest.TestCase):
def _load_datasamples(self, num_samples):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

2
tests/models/phi4_multimodal/test_feature_extractor_phi4_multimodal.py
View File

@@ -207,7 +207,7 @@ class Phi4MultimodalFeatureExtractionTest(SequenceFeatureExtractionTestMixin, un
def _load_datasamples(self, num_samples):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

19
tests/models/phi4_multimodal/test_modeling_phi4_multimodal.py
View File

@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import tempfile
import unittest
import requests
@@ -33,13 +32,13 @@ from transformers import (
from transformers.testing_utils import (
Expectations,
cleanup,
require_soundfile,
require_torch,
require_torch_large_accelerator,
require_torchcodec,
slow,
torch_device,
)
from transformers.utils import is_soundfile_available
from transformers.utils import is_torchcodec_available
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
@@ -54,8 +53,8 @@ if is_vision_available():
from PIL import Image
if is_soundfile_available():
import soundfile
if is_torchcodec_available():
import torchcodec
class Phi4MultimodalModelTester:
@@ -296,11 +295,9 @@ class Phi4MultimodalIntegrationTest(unittest.TestCase):
self.assistant_token = "<|assistant|>"
self.end_token = "<|end|>"
self.image = Image.open(requests.get(self.image_url, stream=True).raw)
with tempfile.NamedTemporaryFile(mode="w+b", suffix=".wav") as tmp:
tmp.write(requests.get(self.audio_url, stream=True).raw.data)
tmp.flush()
tmp.seek(0)
self.audio, self.sampling_rate = soundfile.read(tmp.name)
audio_bytes = requests.get(self.audio_url, stream=True).raw.data
samples = torchcodec.decoders.AudioDecoder(audio_bytes).get_all_samples()
self.audio, self.sampling_rate = samples.data, samples.sample_rate
cleanup(torch_device, gc_collect=True)
@@ -378,7 +375,7 @@ class Phi4MultimodalIntegrationTest(unittest.TestCase):
self.assertEqual(response, EXPECTED_RESPONSE)
@require_soundfile
@require_torchcodec
def test_audio_text_generation(self):
model = AutoModelForCausalLM.from_pretrained(
self.checkpoint_path, revision=self.revision, torch_dtype=torch.float16, device_map=torch_device

4
tests/models/sew/test_modeling_sew.py
View File

@@ -19,7 +19,7 @@ import unittest
import pytest
from transformers import SEWConfig, is_torch_available
from transformers.testing_utils import require_soundfile, require_torch, slow, torch_device
from transformers.testing_utils import require_torch, require_torchcodec, slow, torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
@@ -453,7 +453,7 @@ class SEWUtilsTest(unittest.TestCase):
@require_torch
@require_soundfile
@require_torchcodec
@slow
class SEWModelIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):

4
tests/models/sew_d/test_modeling_sew_d.py
View File

@@ -19,7 +19,7 @@ import unittest
import pytest
from transformers import SEWDConfig, is_torch_available
from transformers.testing_utils import require_soundfile, require_torch, slow, torch_device
from transformers.testing_utils import require_torch, require_torchcodec, slow, torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
@@ -464,7 +464,7 @@ class SEWDUtilsTest(unittest.TestCase):
@require_torch
@require_soundfile
@require_torchcodec
@slow
class SEWDModelIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):

2
tests/models/speech_to_text/test_feature_extraction_speech_to_text.py
View File

@@ -294,7 +294,7 @@ class Speech2TextFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unitt
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

2
tests/models/speecht5/test_feature_extraction_speecht5.py
View File

@@ -381,7 +381,7 @@ class SpeechT5FeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

4
tests/models/speecht5/test_modeling_speecht5.py
View File

@@ -764,7 +764,7 @@ class SpeechT5ForSpeechToTextIntegrationTests(unittest.TestCase):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]
@@ -1792,7 +1792,7 @@ class SpeechT5ForSpeechToSpeechIntegrationTests(unittest.TestCase):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

4
tests/models/unispeech/test_modeling_unispeech.py
View File

@@ -21,7 +21,7 @@ import pytest
from datasets import load_dataset
from transformers import UniSpeechConfig, is_torch_available
from transformers.testing_utils import is_flaky, require_soundfile, require_torch, slow, torch_device
from transformers.testing_utils import is_flaky, require_torch, require_torchcodec, slow, torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
@@ -553,7 +553,7 @@ class UniSpeechRobustModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.T
@require_torch
@require_soundfile
@require_torchcodec
@slow
class UniSpeechModelIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):

4
tests/models/unispeech_sat/test_modeling_unispeech_sat.py
View File

@@ -21,7 +21,7 @@ import pytest
from datasets import load_dataset
from transformers import UniSpeechSatConfig, is_torch_available
from transformers.testing_utils import require_soundfile, require_torch, slow, torch_device
from transformers.testing_utils import require_torch, require_torchcodec, slow, torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
@@ -807,7 +807,7 @@ class UniSpeechSatRobustModelTest(ModelTesterMixin, unittest.TestCase):
@require_torch
@require_soundfile
@require_torchcodec
@slow
class UniSpeechSatModelIntegrationTest(unittest.TestCase):
def _load_datasamples(self, num_samples):

2
tests/models/univnet/test_feature_extraction_univnet.py
View File

@@ -330,7 +330,7 @@ class UnivNetFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
ds = ds.cast_column("audio", Audio(sampling_rate=self.feat_extract_tester.sampling_rate))
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples], [x["sampling_rate"] for x in speech_samples]

2
tests/models/univnet/test_modeling_univnet.py
View File

@@ -216,7 +216,7 @@ class UnivNetModelIntegrationTests(unittest.TestCase):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
ds = ds.cast_column("audio", Audio(sampling_rate=sampling_rate))
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples], [x["sampling_rate"] for x in speech_samples]

4
tests/models/wav2vec2/test_modeling_wav2vec2.py
View File

@@ -34,10 +34,10 @@ from transformers.testing_utils import (
is_torchaudio_available,
require_flash_attn,
require_pyctcdecode,
require_soundfile,
require_torch,
require_torch_gpu,
require_torchaudio,
require_torchcodec,
run_test_in_subprocess,
slow,
torch_device,
@@ -1444,7 +1444,7 @@ class Wav2Vec2UtilsTest(unittest.TestCase):
@require_torch
@require_soundfile
@require_torchcodec
@slow
class Wav2Vec2ModelIntegrationTest(unittest.TestCase):
def tearDown(self):

2
tests/models/whisper/test_feature_extraction_whisper.py
View File

@@ -254,7 +254,7 @@ class WhisperFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.
def _load_datasamples(self, num_samples):
ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
# automatic decoding with librispeech
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]

2
tests/models/whisper/test_modeling_whisper.py
View File

@@ -1460,7 +1460,7 @@ class WhisperModelIntegrationTests(unittest.TestCase):
def _load_datasamples(self, num_samples):
self._load_dataset()
ds = self._dataset
speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
speech_samples = ds.sort("id")[:num_samples]["audio"]
return [x["array"] for x in speech_samples]
@slow