[Pipeline] Add zero shot audio classificatoin pipeline (#21600)

* add pipeline * update init * add zero shot to init * update inits and correct checkpoints * update base to support input features * add tests * Update src/transformers/pipelines/zero_shot_audio_classification.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * Update src/transformers/pipelines/zero_shot_audio_classification.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * update pieline code * use tiny checkpoint * nits and expected value with tiny model * style * last nit on tests values * fix styling * fix collate fn that was casting t float * update --------- Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com>
2023-02-27 11:43:44 +01:00
parent 2ea1ef9090
commit cc44e72d14
7 changed files with 267 additions and 4 deletions
--- a/tests/pipelines/test_pipelines_zero_shot_audio_classification.py
+++ b/tests/pipelines/test_pipelines_zero_shot_audio_classification.py
@@ -0,0 +1,95 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from datasets import load_dataset
+
+from transformers.pipelines import pipeline
+from transformers.testing_utils import nested_simplify, require_torch, slow
+
+from .test_pipelines_common import PipelineTestCaseMeta
+
+
+@require_torch
+class ZeroShotAudioClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
+    # Deactivating auto tests since we don't have a good MODEL_FOR_XX mapping,
+    # and only CLAP would be there for now.
+    # model_mapping = {CLAPConfig: CLAPModel}
+
+    @require_torch
+    def test_small_model_pt(self):
+        audio_classifier = pipeline(
+            task="zero-shot-audio-classification", model="hf-internal-testing/tiny-clap-htsat-unfused"
+        )
+        dataset = load_dataset("ashraq/esc50")
+        audio = dataset["train"]["audio"][-1]["array"]
+        output = audio_classifier(audio, candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"])
+        self.assertEqual(
+            nested_simplify(output),
+            [{"score": 0.501, "label": "Sound of a dog"}, {"score": 0.499, "label": "Sound of vaccum cleaner"}],
+        )
+
+    @unittest.skip("No models are available in TF")
+    def test_small_model_tf(self):
+        pass
+
+    @slow
+    @require_torch
+    def test_large_model_pt(self):
+        audio_classifier = pipeline(
+            task="zero-shot-audio-classification",
+            model="laion/clap-htsat-unfused",
+        )
+        # This is an audio of a dog
+        dataset = load_dataset("ashraq/esc50")
+        audio = dataset["train"]["audio"][-1]["array"]
+        output = audio_classifier(audio, candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"])
+
+        self.assertEqual(
+            nested_simplify(output),
+            [
+                {"score": 0.999, "label": "Sound of a dog"},
+                {"score": 0.001, "label": "Sound of vaccum cleaner"},
+            ],
+        )
+
+        output = audio_classifier([audio] * 5, candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"])
+        self.assertEqual(
+            nested_simplify(output),
+            [
+                [
+                    {"score": 0.999, "label": "Sound of a dog"},
+                    {"score": 0.001, "label": "Sound of vaccum cleaner"},
+                ],
+            ]
+            * 5,
+        )
+        output = audio_classifier(
+            [audio] * 5, candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"], batch_size=5
+        )
+        self.assertEqual(
+            nested_simplify(output),
+            [
+                [
+                    {"score": 0.999, "label": "Sound of a dog"},
+                    {"score": 0.001, "label": "Sound of vaccum cleaner"},
+                ],
+            ]
+            * 5,
+        )
+
+    @unittest.skip("No models are available in TF")
+    def test_large_model_tf(self):
+        pass