diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index bd96c31c51..2e87b4c2e1 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -171,6 +171,7 @@ class CircleCIJob: "command": f"TESTS=$(circleci tests split --split-by=timings {self.job_name}_test_list.txt) && echo $TESTS > splitted_tests.txt && echo $TESTS | tr ' ' '\n'" if self.parallelism else f"awk '{{printf \"%s \", $0}}' {self.job_name}_test_list.txt > splitted_tests.txt" } }, + {"run": {"name": "fetch hub objects before pytest", "command": "python3 utils/fetch_hub_objects_for_ci.py"}}, {"run": { "name": "Run tests", "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n {self.pytest_num_workers} {junit_flags} {repeat_on_failure_flags} {' '.join(pytest_flags)} $(cat splitted_tests.txt) | tee tests_output.txt)"} diff --git a/tests/pipelines/test_pipelines_audio_classification.py b/tests/pipelines/test_pipelines_audio_classification.py index db2d3b3b59..08a361add7 100644 --- a/tests/pipelines/test_pipelines_audio_classification.py +++ b/tests/pipelines/test_pipelines_audio_classification.py @@ -14,6 +14,7 @@ import unittest +import datasets import numpy as np from huggingface_hub import AudioClassificationOutputElement @@ -24,6 +25,7 @@ from transformers import ( ) from transformers.pipelines import AudioClassificationPipeline, pipeline from transformers.testing_utils import ( + _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -45,6 +47,9 @@ class AudioClassificationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING + if _run_pipeline_tests: + _dataset = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + def get_test_pipeline( self, model, @@ -94,11 +99,8 @@ class AudioClassificationPipelineTests(unittest.TestCase): @require_torchaudio def run_torchaudio(self, audio_classifier): - import datasets - # test with a local file - dataset = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - audio = dataset[0]["audio"]["array"] + audio = self._dataset[0]["audio"]["array"] output = audio_classifier(audio) self.assertEqual( output, @@ -168,8 +170,6 @@ class AudioClassificationPipelineTests(unittest.TestCase): @require_torch @slow def test_large_model_pt(self): - import datasets - model = "superb/wav2vec2-base-superb-ks" audio_classifier = pipeline("audio-classification", model=model) diff --git a/tests/pipelines/test_pipelines_depth_estimation.py b/tests/pipelines/test_pipelines_depth_estimation.py index a905aa8169..73176a4383 100644 --- a/tests/pipelines/test_pipelines_depth_estimation.py +++ b/tests/pipelines/test_pipelines_depth_estimation.py @@ -14,12 +14,14 @@ import unittest +import datasets from huggingface_hub import DepthEstimationOutput from huggingface_hub.utils import insecure_hashlib from transformers import MODEL_FOR_DEPTH_ESTIMATION_MAPPING, is_torch_available, is_vision_available from transformers.pipelines import DepthEstimationPipeline, pipeline from transformers.testing_utils import ( + _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -58,6 +60,13 @@ def hashimage(image: Image) -> str: class DepthEstimationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_DEPTH_ESTIMATION_MAPPING + if _run_pipeline_tests: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + _dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) + def get_test_pipeline( self, model, @@ -83,21 +92,17 @@ class DepthEstimationPipelineTests(unittest.TestCase): def run_pipeline_test(self, depth_estimator, examples): outputs = depth_estimator("./tests/fixtures/tests_samples/COCO/000000039769.png") self.assertEqual({"predicted_depth": ANY(torch.Tensor), "depth": ANY(Image.Image)}, outputs) - import datasets - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1") outputs = depth_estimator( [ Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), "http://images.cocodataset.org/val2017/000000039769.jpg", # RGBA - dataset[0]["image"], + self._dataset[0]["image"], # LA - dataset[1]["image"], + self._dataset[1]["image"], # L - dataset[2]["image"], + self._dataset[2]["image"], ] ) self.assertEqual( diff --git a/tests/pipelines/test_pipelines_image_classification.py b/tests/pipelines/test_pipelines_image_classification.py index 5f926dd2f0..e5dce2a03d 100644 --- a/tests/pipelines/test_pipelines_image_classification.py +++ b/tests/pipelines/test_pipelines_image_classification.py @@ -14,6 +14,7 @@ import unittest +import datasets from huggingface_hub import ImageClassificationOutputElement from transformers import ( @@ -25,6 +26,7 @@ from transformers import ( ) from transformers.pipelines import ImageClassificationPipeline, pipeline from transformers.testing_utils import ( + _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -58,6 +60,13 @@ class ImageClassificationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING + if _run_pipeline_tests: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + _dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) + def get_test_pipeline( self, model, @@ -93,23 +102,17 @@ class ImageClassificationPipelineTests(unittest.TestCase): ], ) - import datasets - - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1") - # Accepts URL + PIL.Image + lists outputs = image_classifier( [ Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), "http://images.cocodataset.org/val2017/000000039769.jpg", # RGBA - dataset[0]["image"], + self._dataset[0]["image"], # LA - dataset[1]["image"], + self._dataset[1]["image"], # L - dataset[2]["image"], + self._dataset[2]["image"], ] ) self.assertEqual( diff --git a/tests/pipelines/test_pipelines_image_segmentation.py b/tests/pipelines/test_pipelines_image_segmentation.py index 44ed4e5453..af251230a6 100644 --- a/tests/pipelines/test_pipelines_image_segmentation.py +++ b/tests/pipelines/test_pipelines_image_segmentation.py @@ -37,6 +37,7 @@ from transformers import ( pipeline, ) from transformers.testing_utils import ( + _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -89,6 +90,13 @@ class ImageSegmentationPipelineTests(unittest.TestCase): + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else []) ) + if _run_pipeline_tests: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + _dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) + def get_test_pipeline( self, model, @@ -130,20 +138,22 @@ class ImageSegmentationPipelineTests(unittest.TestCase): # to make it work self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * n, outputs) - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1") - # RGBA - outputs = image_segmenter(dataset[0]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0) + outputs = image_segmenter( + self._dataset[0]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0 + ) m = len(outputs) self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * m, outputs) # LA - outputs = image_segmenter(dataset[1]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0) + outputs = image_segmenter( + self._dataset[1]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0 + ) m = len(outputs) self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * m, outputs) # L - outputs = image_segmenter(dataset[2]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0) + outputs = image_segmenter( + self._dataset[2]["image"], threshold=0.0, mask_threshold=0, overlap_mask_area_threshold=0 + ) m = len(outputs) self.assertEqual([{"score": ANY(float, type(None)), "label": ANY(str), "mask": ANY(Image.Image)}] * m, outputs) diff --git a/tests/pipelines/test_pipelines_object_detection.py b/tests/pipelines/test_pipelines_object_detection.py index d58f3749ec..c6fa2c1ec1 100644 --- a/tests/pipelines/test_pipelines_object_detection.py +++ b/tests/pipelines/test_pipelines_object_detection.py @@ -14,6 +14,7 @@ import unittest +import datasets from huggingface_hub import ObjectDetectionOutputElement from transformers import ( @@ -25,6 +26,7 @@ from transformers import ( pipeline, ) from transformers.testing_utils import ( # + _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -56,6 +58,13 @@ else: class ObjectDetectionPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_OBJECT_DETECTION_MAPPING + if _run_pipeline_tests: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + _dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) + def get_test_pipeline( self, model, @@ -89,21 +98,15 @@ class ObjectDetectionPipelineTests(unittest.TestCase): }, ) - import datasets - - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - dataset = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1") - batch = [ Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"), "http://images.cocodataset.org/val2017/000000039769.jpg", # RGBA - dataset[0]["image"], + self._dataset[0]["image"], # LA - dataset[1]["image"], + self._dataset[1]["image"], # L - dataset[2]["image"], + self._dataset[2]["image"], ] batch_outputs = object_detector(batch, threshold=0.0) diff --git a/utils/fetch_hub_objects_for_ci.py b/utils/fetch_hub_objects_for_ci.py new file mode 100644 index 0000000000..3975921a84 --- /dev/null +++ b/utils/fetch_hub_objects_for_ci.py @@ -0,0 +1,9 @@ +from transformers.testing_utils import _run_pipeline_tests + + +if __name__ == "__main__": + if _run_pipeline_tests: + import datasets + + _ = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + _ = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")