Au revoir flaky test_fast_is_faster_than_slow (#36240)

* fix

* fix

* fix

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/rt_detr/test_image_processing_rt_detr.py

@@ -16,7 +16,14 @@ import unittest
 
 import requests
 
-from transformers.testing_utils import require_torch, require_torch_gpu, require_torchvision, require_vision, slow
+from transformers.testing_utils import (
+    is_flaky,
+    require_torch,
+    require_torch_gpu,
+    require_torchvision,
+    require_vision,
+    slow,
+)
 from transformers.utils import is_torch_available, is_torchvision_available, is_vision_available
 
 from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
@@ -427,3 +434,9 @@ class RtDetrImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
         )
         # verify size
         torch.testing.assert_close(encoding_cpu["labels"][0]["size"], encoding_gpu["labels"][0]["size"].to("cpu"))
+
+    @is_flaky(
+        description="Still flaky with a failing ratio of ~0.6% after #36240",
+    )
+    def test_fast_is_faster_than_slow(self):
+        super().test_fast_is_faster_than_slow()

tests/test_image_processing_common.py

@@ -223,9 +223,14 @@ class ImageProcessingTestMixin:
             # Warmup
             for _ in range(5):
                 _ = image_processor(image, return_tensors="pt")
+            all_times = []
+            for _ in range(10):
                 start = time.time()
                 _ = image_processor(image, return_tensors="pt")
-            return time.time() - start
+                all_times.append(time.time() - start)
+            # Take the average of the fastest 3 runs
+            avg_time = sum(sorted(all_times[:3])) / 3.0
+            return avg_time
 
         dummy_images = torch.randint(0, 255, (4, 3, 224, 224), dtype=torch.uint8)
         image_processor_slow = self.image_processing_class(**self.image_processor_dict)