Avoid pipeline test failing related to Hub call (#37170)

* cls

* cls

* cls

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

.circleci/create_circleci_config.py

@@ -171,6 +171,7 @@ class CircleCIJob:
                     "command": f"TESTS=$(circleci tests split  --split-by=timings {self.job_name}_test_list.txt) && echo $TESTS > splitted_tests.txt && echo $TESTS | tr ' ' '\n'" if self.parallelism else f"awk '{{printf \"%s \", $0}}' {self.job_name}_test_list.txt > splitted_tests.txt"
                     }
             },
+            {"run": {"name": "fetch hub objects before pytest", "command": "python3 utils/fetch_hub_objects_for_ci.py"}},
             {"run": {
                 "name": "Run tests",
                 "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n {self.pytest_num_workers} {junit_flags} {repeat_on_failure_flags} {' '.join(pytest_flags)} $(cat splitted_tests.txt) | tee tests_output.txt)"}

tests/pipelines/test_pipelines_audio_classification.py

@@ -14,6 +14,7 @@
 
 import unittest
 
+import datasets
 import numpy as np
 from huggingface_hub import AudioClassificationOutputElement
 
@@ -24,6 +25,7 @@ from transformers import (
 )
 from transformers.pipelines import AudioClassificationPipeline, pipeline
 from transformers.testing_utils import (
+    _run_pipeline_tests,
     compare_pipeline_output_to_hub_spec,
     is_pipeline_test,
     nested_simplify,
@@ -45,6 +47,9 @@ class AudioClassificationPipelineTests(unittest.TestCase):
     model_mapping = MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
     tf_model_mapping = TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
 
+    if _run_pipeline_tests:
+        _dataset = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+
     def get_test_pipeline(
         self,
         model,
@@ -94,11 +99,8 @@ class AudioClassificationPipelineTests(unittest.TestCase):
 
     @require_torchaudio
     def run_torchaudio(self, audio_classifier):
-        import datasets
-
         # test with a local file
-        dataset = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+        audio = self._dataset[0]["audio"]["array"]
-        audio = dataset[0]["audio"]["array"]
         output = audio_classifier(audio)
         self.assertEqual(
             output,
@@ -168,8 +170,6 @@ class AudioClassificationPipelineTests(unittest.TestCase):
     @require_torch
     @slow
     def test_large_model_pt(self):
-        import datasets
-
         model = "superb/wav2vec2-base-superb-ks"
 
         audio_classifier = pipeline("audio-classification", model=model)

tests/pipelines/test_pipelines_depth_estimation.py

@@ -14,12 +14,14 @@
 
 import unittest
 
+import datasets
 from huggingface_hub import DepthEstimationOutput
 from huggingface_hub.utils import insecure_hashlib
 
 from transformers import MODEL_FOR_DEPTH_ESTIMATION_MAPPING, is_torch_available, is_vision_available
 from transformers.pipelines import DepthEstimationPipeline, pipeline
 from transformers.testing_utils import (
+    _run_pipeline_tests,
     compare_pipeline_output_to_hub_spec,
     is_pipeline_test,
     nested_simplify,
@@ -58,6 +60,13 @@ def hashimage(image: Image) -> str:
 class DepthEstimationPipelineTests(unittest.TestCase):
     model_mapping = MODEL_FOR_DEPTH_ESTIMATION_MAPPING
 
+    if _run_pipeline_tests:
+        # we use revision="refs/pr/1" until the PR is merged
+        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
+        _dataset = datasets.load_dataset(
+            "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1"
+        )
+
     def get_test_pipeline(
         self,
         model,
@@ -83,21 +92,17 @@ class DepthEstimationPipelineTests(unittest.TestCase):
     def run_pipeline_test(self, depth_estimator, examples):
         outputs = depth_estimator("./tests/fixtures/tests_samples/COCO/000000039769.png")
         self.assertEqual({"predicted_depth": ANY(torch.Tensor), "depth": ANY(Image.Image)}, outputs)
-        import datasets
 
-        # we use revision="refs/pr/1" until the PR is merged
-        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
-        dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")
         outputs = depth_estimator(
             [
                 Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
                 "http://images.cocodataset.org/val2017/000000039769.jpg",
                 # RGBA
-                dataset[0]["image"],
+                self._dataset[0]["image"],
                 # LA
-                dataset[1]["image"],
+                self._dataset[1]["image"],
                 # L
-                dataset[2]["image"],
+                self._dataset[2]["image"],
             ]
         )
         self.assertEqual(

tests/pipelines/test_pipelines_image_classification.py

@@ -14,6 +14,7 @@
 
 import unittest
 
+import datasets
 from huggingface_hub import ImageClassificationOutputElement
 
 from transformers import (
@@ -25,6 +26,7 @@ from transformers import (
 )
 from transformers.pipelines import ImageClassificationPipeline, pipeline
 from transformers.testing_utils import (
+    _run_pipeline_tests,
     compare_pipeline_output_to_hub_spec,
     is_pipeline_test,
     nested_simplify,
@@ -58,6 +60,13 @@ class ImageClassificationPipelineTests(unittest.TestCase):
     model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
     tf_model_mapping = TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
 
+    if _run_pipeline_tests:
+        # we use revision="refs/pr/1" until the PR is merged
+        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
+        _dataset = datasets.load_dataset(
+            "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1"
+        )
+
     def get_test_pipeline(
         self,
         model,
@@ -93,23 +102,17 @@ class ImageClassificationPipelineTests(unittest.TestCase):
             ],
         )
 
-        import datasets
-
-        # we use revision="refs/pr/1" until the PR is merged
-        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
-        dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")
-
         # Accepts URL + PIL.Image + lists
         outputs = image_classifier(
             [
                 Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
                 "http://images.cocodataset.org/val2017/000000039769.jpg",
                 # RGBA
-                dataset[0]["image"],
+                self._dataset[0]["image"],
                 # LA
-                dataset[1]["image"],
+                self._dataset[1]["image"],
                 # L
-                dataset[2]["image"],
+                self._dataset[2]["image"],
             ]
         )
         self.assertEqual(

tests/pipelines/test_pipelines_image_segmentation.py

@@ -37,6 +37,7 @@ from transformers import (
     pipeline,
 )
 from transformers.testing_utils import (
+    _run_pipeline_tests,
     compare_pipeline_output_to_hub_spec,
     is_pipeline_test,
     nested_simplify,
@@ -89,6 +90,13 @@ class ImageSegmentationPipelineTests(unittest.TestCase):
         + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else [])
     )
 
+    if _run_pipeline_tests:
+        # we use revision="refs/pr/1" until the PR is merged
+        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
+        _dataset = datasets.load_dataset(
+            "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1"
+        )
+
     def get_test_pipeline(
         self,
         model,
@@ -130,20 +138,22 @@ class ImageSegmentationPipelineTests(unittest.TestCase):
         # to make it work
         self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * n, outputs)
 
-        # we use revision="refs/pr/1" until the PR is merged
-        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
-        dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")
-
         # RGBA
-        outputs = image_segmenter(dataset[0]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0)
+        outputs = image_segmenter(
+            self._dataset[0]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0
+        )
         m = len(outputs)
         self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * m, outputs)
         # LA
-        outputs = image_segmenter(dataset[1]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0)
+        outputs = image_segmenter(
+            self._dataset[1]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0
+        )
         m = len(outputs)
         self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * m, outputs)
         # L
-        outputs = image_segmenter(dataset[2]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0)
+        outputs = image_segmenter(
+            self._dataset[2]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0
+        )
         m = len(outputs)
         self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * m, outputs)
 

tests/pipelines/test_pipelines_object_detection.py

@@ -14,6 +14,7 @@
 
 import unittest
 
+import datasets
 from huggingface_hub import ObjectDetectionOutputElement
 
 from transformers import (
@@ -25,6 +26,7 @@ from transformers import (
     pipeline,
 )
 from transformers.testing_utils import (  #
+    _run_pipeline_tests,
     compare_pipeline_output_to_hub_spec,
     is_pipeline_test,
     nested_simplify,
@@ -56,6 +58,13 @@ else:
 class ObjectDetectionPipelineTests(unittest.TestCase):
     model_mapping = MODEL_FOR_OBJECT_DETECTION_MAPPING
 
+    if _run_pipeline_tests:
+        # we use revision="refs/pr/1" until the PR is merged
+        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
+        _dataset = datasets.load_dataset(
+            "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1"
+        )
+
     def get_test_pipeline(
         self,
         model,
@@ -89,21 +98,15 @@ class ObjectDetectionPipelineTests(unittest.TestCase):
                 },
             )
 
-        import datasets
-
-        # we use revision="refs/pr/1" until the PR is merged
-        # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1
-        dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")
-
         batch = [
             Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"),
             "http://images.cocodataset.org/val2017/000000039769.jpg",
             # RGBA
-            dataset[0]["image"],
+            self._dataset[0]["image"],
             # LA
-            dataset[1]["image"],
+            self._dataset[1]["image"],
             # L
-            dataset[2]["image"],
+            self._dataset[2]["image"],
         ]
         batch_outputs = object_detector(batch, threshold=0.0)
 

utils/fetch_hub_objects_for_ci.py

@@ -0,0 +1,9 @@
+from transformers.testing_utils import _run_pipeline_tests
+
+
+if __name__ == "__main__":
+    if _run_pipeline_tests:
+        import datasets
+
+        _ = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+        _ = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")