Add a decorator for flaky tests (#19498)

* Add a decorator for flaky tests

* Quality

* Don't break the rest

* Address review comments

* Fix test name

* Fix typo and print to stderr

tests/models/time_series_transformer/test_modeling_time_series_transformer.py

@@ -20,7 +20,7 @@ import unittest
 
 from huggingface_hub import hf_hub_download
 from transformers import is_torch_available
-from transformers.testing_utils import require_torch, slow, torch_device
+from transformers.testing_utils import is_flaky, require_torch, slow, torch_device
 
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
@@ -359,6 +359,10 @@ class TimeSeriesTransformerModelTest(ModelTesterMixin, unittest.TestCase):
             [self.model_tester.num_attention_heads, encoder_seq_length, encoder_seq_length],
         )
 
+    @is_flaky()
+    def test_retain_grad_hidden_states_attentions(self):
+        super().test_retain_grad_hidden_states_attentions()
+
 
 def prepare_batch(filename="train-batch.pt"):
     file = hf_hub_download(repo_id="kashif/tourism-monthly-batch", filename=filename, repo_type="dataset")

tests/models/wav2vec2/test_modeling_flax_wav2vec2.py

@@ -21,7 +21,9 @@ from datasets import load_dataset
 
 from transformers import Wav2Vec2Config, is_flax_available
 from transformers.testing_utils import (
+    is_flaky,
     is_librosa_available,
+    is_pt_flax_cross_test,
     is_pyctcdecode_available,
     require_flax,
     require_librosa,
@@ -302,6 +304,11 @@ class FlaxWav2Vec2ModelTest(FlaxModelTesterMixin, unittest.TestCase):
             outputs = model(np.ones((1, 1024), dtype="f4"))
             self.assertIsNotNone(outputs)
 
+    @is_pt_flax_cross_test
+    @is_flaky()
+    def test_equivalence_pt_to_flax(self):
+        super().test_equivalence_pt_to_flax()
+
 
 @require_flax
 class FlaxWav2Vec2UtilsTest(unittest.TestCase):

tests/models/wav2vec2/test_modeling_tf_wav2vec2.py

@@ -26,7 +26,7 @@ from datasets import load_dataset
 
 from huggingface_hub import snapshot_download
 from transformers import Wav2Vec2Config, is_tf_available
-from transformers.testing_utils import require_librosa, require_pyctcdecode, require_tf, slow
+from transformers.testing_utils import is_flaky, require_librosa, require_pyctcdecode, require_tf, slow
 from transformers.utils import is_librosa_available, is_pyctcdecode_available
 
 from ...test_configuration_common import ConfigTester
@@ -309,6 +309,7 @@ class TFWav2Vec2ModelTest(TFModelTesterMixin, unittest.TestCase):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.check_ctc_loss(*config_and_inputs)
 
+    @is_flaky()
     def test_labels_out_of_vocab(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.check_labels_out_of_vocab(*config_and_inputs)