diff --git a/tests/test_modeling_hubert.py b/tests/test_modeling_hubert.py
index 5d3d27721e..ad8a1dd206 100644
--- a/tests/test_modeling_hubert.py
+++ b/tests/test_modeling_hubert.py
@@ -615,7 +615,9 @@ class HubertModelIntegrationTest(unittest.TestCase):
 
         ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
         # automatic decoding with librispeech
-        speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]
+        speech_samples = ds.sort("id").filter(
+            lambda x: x["id"] in [f"1272-141231-000{i}" for i in range(num_samples)]
+        )[:num_samples]["audio"]
 
         return [x["array"] for x in speech_samples]