[CI] Check test if the GenerationTesterMixin inheritance is correct 🐛 🔫 (#36180)

This commit is contained in:
Joao Gante
2025-02-21 10:18:20 +00:00
committed by GitHub
parent a957b7911a
commit 678885bbbd
39 changed files with 180 additions and 68 deletions

View File

@@ -451,6 +451,8 @@ class BigBirdModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase)
if is_torch_available()
else ()
)
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": BigBirdModel,

View File

@@ -799,6 +799,8 @@ class BlipVQAModelTester:
@require_vision
class BlipVQAModelTest(ModelTesterMixin, unittest.TestCase):
all_model_classes = (BlipForQuestionAnswering,) if is_torch_available() else ()
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
fx_compatible = False
test_head_masking = False
test_pruning = False
@@ -1106,6 +1108,8 @@ class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase):
@require_torch
class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase):
all_model_classes = (BlipForConditionalGeneration,) if is_torch_available() else ()
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
fx_compatible = False
test_head_masking = False
test_pruning = False

View File

@@ -885,7 +885,7 @@ class Blip2ModelTester:
@require_torch
class Blip2ModelTest(ModelTesterMixin, PipelineTesterMixin, GenerationTesterMixin, unittest.TestCase):
class Blip2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (Blip2ForConditionalGeneration, Blip2Model) if is_torch_available() else ()
# Doesn't run generation tests. TODO: fix generation tests for Blip2ForConditionalGeneration
all_generative_model_classes = ()

View File

@@ -408,6 +408,8 @@ class ClvpModelForConditionalGenerationTester:
@require_torch
class ClvpModelForConditionalGenerationTest(ModelTesterMixin, unittest.TestCase):
all_model_classes = (ClvpModelForConditionalGeneration,) if is_torch_available() else ()
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
test_head_masking = False
test_pruning = False

View File

@@ -22,7 +22,6 @@ from transformers import ConditionalDetrConfig, ResNetConfig, is_torch_available
from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -173,7 +172,7 @@ class ConditionalDetrModelTester:
@require_torch
class ConditionalDetrModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class ConditionalDetrModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (
(
ConditionalDetrModel,

View File

@@ -136,6 +136,8 @@ class CpmAntModelTester:
@require_torch
class CpmAntModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{"feature-extraction": CpmAntModel, "text-generation": CpmAntForCausalLM} if is_torch_available() else {}
)

View File

@@ -23,7 +23,6 @@ from transformers import DabDetrConfig, ResNetConfig, is_torch_available, is_vis
from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -174,15 +173,8 @@ class DabDetrModelTester:
@require_torch
class DabDetrModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (
(
DabDetrModel,
DabDetrForObjectDetection,
)
if is_torch_available()
else ()
)
class DabDetrModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (DabDetrModel, DabDetrForObjectDetection) if is_torch_available() else ()
pipeline_model_mapping = (
{
"image-feature-extraction": DabDetrModel,

View File

@@ -20,7 +20,6 @@ import unittest
from transformers import DecisionTransformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -125,7 +124,7 @@ class DecisionTransformerModelTester:
@require_torch
class DecisionTransformerModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class DecisionTransformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (DecisionTransformerModel,) if is_torch_available() else ()
pipeline_model_mapping = {"feature-extraction": DecisionTransformerModel} if is_torch_available() else {}

View File

@@ -31,7 +31,6 @@ from transformers.testing_utils import (
torch_device,
)
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -188,7 +187,7 @@ class DeformableDetrModelTester:
@require_torch
class DeformableDetrModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class DeformableDetrModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (DeformableDetrModel, DeformableDetrForObjectDetection) if is_torch_available() else ()
pipeline_model_mapping = (
{"image-feature-extraction": DeformableDetrModel, "object-detection": DeformableDetrForObjectDetection}

View File

@@ -22,7 +22,6 @@ from transformers import DetrConfig, ResNetConfig, is_torch_available, is_vision
from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -169,7 +168,7 @@ class DetrModelTester:
@require_torch
class DetrModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class DetrModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (
(
DetrModel,

View File

@@ -389,6 +389,8 @@ class ElectraModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase)
if is_torch_available()
else ()
)
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": ElectraModel,

View File

@@ -377,6 +377,8 @@ class FlaubertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase
if is_torch_available()
else ()
)
# Doesn't run generation tests. Outdated custom `prepare_inputs_for_generation` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": FlaubertModel,

View File

@@ -326,6 +326,8 @@ class IdeficsModelTester:
@require_torch
class IdeficsModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (IdeficsModel, IdeficsForVisionText2Text) if is_torch_available() else ()
# Doesn't run generation tests here -- idefics has a dedicated tester for generation tests below
all_generative_model_classes = ()
pipeline_model_mapping = (
{"feature-extraction": IdeficsModel, "image-text-to-text": IdeficsForVisionText2Text}
if is_torch_available()
@@ -868,6 +870,12 @@ class IdeficsForVisionText2TextTest(IdeficsModelTest, GenerationTesterMixin, uni
def test_sdpa_can_dispatch_non_composite_models(self):
pass
@unittest.skip(
"Idefics has a separate test runner for generation tests with complex inheritance, causing this check to fail"
)
def test_generation_tester_mixin_inheritance(self):
pass
@require_torch
@require_vision

View File

@@ -19,7 +19,6 @@ import unittest
from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -218,7 +217,7 @@ class LiltModelTester:
@require_torch
class LiltModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class LiltModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (
(
LiltModel,

View File

@@ -282,6 +282,8 @@ class MegatronBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Test
if is_torch_available()
else ()
)
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": MegatronBertModel,

View File

@@ -29,7 +29,6 @@ from transformers.testing_utils import (
torch_device,
)
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -216,7 +215,7 @@ class ModernBertModelTester:
@require_torch
class ModernBertModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class ModernBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
test_torchscript = False
all_model_classes = (

View File

@@ -20,7 +20,6 @@ import unittest
from transformers import MoonshineConfig, is_torch_available
from transformers.testing_utils import cleanup, require_torch, slow, torch_device
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
ModelTesterMixin,
@@ -168,7 +167,7 @@ class MoonshineModelTester:
@require_torch
class MoonshineModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class MoonshineModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (MoonshineModel, MoonshineForConditionalGeneration) if is_torch_available() else ()
# Doesn't run generation tests. TODO (eustache): remove this line and then make CI green
all_generative_model_classes = ()

View File

@@ -672,6 +672,15 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste
self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases))
@unittest.skip(
reason=(
"MusicGen has a custom set of generation tests that rely on `GenerationTesterMixin`, controlled by "
"`greedy_sample_model_classes`"
)
)
def test_generation_tester_mixin_inheritance(self):
pass
def prepare_musicgen_inputs_dict(
config,
@@ -1763,6 +1772,15 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin,
self.assertTrue(all(audio_encoder_grads))
self.assertFalse(all(text_encoder_grads))
@unittest.skip(
reason=(
"MusicGen has a custom set of generation tests that rely on `GenerationTesterMixin`, controlled by "
"`greedy_sample_model_classes`"
)
)
def test_generation_tester_mixin_inheritance(self):
pass
def get_bip_bip(bip_duration=0.125, duration=0.5, sample_rate=32000):
"""Produces a series of 'bip bip' sounds at a given frequency."""

View File

@@ -689,6 +689,15 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes
self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases))
@unittest.skip(
reason=(
"MusicGen has a custom set of generation tests that rely on `GenerationTesterMixin`, controlled by "
"`greedy_sample_model_classes`"
)
)
def test_generation_tester_mixin_inheritance(self):
pass
def prepare_musicgen_melody_inputs_dict(
config,
@@ -1741,6 +1750,15 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester
self.assertTrue(all(audio_encoder_grads))
self.assertFalse(all(text_encoder_grads))
@unittest.skip(
reason=(
"MusicGen has a custom set of generation tests that rely on `GenerationTesterMixin`, controlled by "
"`greedy_sample_model_classes`"
)
)
def test_generation_tester_mixin_inheritance(self):
pass
# Copied from tests.models.musicgen.test_modeling_musicgen.get_bip_bip
def get_bip_bip(bip_duration=0.125, duration=0.5, sample_rate=32000):

View File

@@ -847,7 +847,7 @@ class PegasusXStandaloneDecoderModelTester:
@require_torch
class PegasusXStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
class PegasusXStandaloneDecoderModelTest(ModelTesterMixin, unittest.TestCase):
all_model_classes = (PegasusXDecoder,) if is_torch_available() else ()
test_pruning = False
is_encoder_decoder = False

View File

@@ -33,7 +33,6 @@ from transformers.testing_utils import (
)
from transformers.utils import is_essentia_available, is_librosa_available, is_scipy_available, is_torch_available
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -504,7 +503,7 @@ class Pop2PianoModelTester:
@require_torch
class Pop2PianoModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class Pop2PianoModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (Pop2PianoForConditionalGeneration,) if is_torch_available() else ()
# Doesn't run generation tests. Has custom generation method with a different interface
all_generative_model_classes = ()

View File

@@ -152,6 +152,8 @@ class Qwen2AudioForConditionalGenerationModelTest(ModelTesterMixin, unittest.Tes
"""
all_model_classes = (Qwen2AudioForConditionalGeneration,) if is_torch_available() else ()
# Doesn't run generation tests. TODO eustache/joao: some generation tests are broken, the errors seem cache-related
all_generative_model_classes = ()
test_pruning = False
test_head_masking = False
_is_composite = True

View File

@@ -26,7 +26,6 @@ from transformers.testing_utils import (
torch_device,
)
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -281,7 +280,7 @@ class RecurrentGemmaModelTester:
@require_torch
class RecurrentGemmaModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class RecurrentGemmaModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (RecurrentGemmaForCausalLM,) if is_torch_available() else ()
# Doesn't run generation tests. TODO @gante not fully supported
all_generative_model_classes = ()

View File

@@ -373,6 +373,8 @@ class RemBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase)
if is_torch_available()
else ()
)
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": RemBertModel,

View File

@@ -570,6 +570,8 @@ class RoCBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase)
if is_torch_available()
else ()
)
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": RoCBertModel,

View File

@@ -392,6 +392,8 @@ class RoFormerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase
if is_torch_available()
else ()
)
# Doesn't run generation tests. There are interface mismatches when using `generate` -- TODO @gante
all_generative_model_classes = ()
pipeline_model_mapping = (
{
"feature-extraction": RoFormerModel,

View File

@@ -23,7 +23,6 @@ from transformers.testing_utils import require_torch, slow, torch_device
from transformers.trainer_utils import set_seed
from transformers.utils import cached_property
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
ModelTesterMixin,
@@ -358,6 +357,8 @@ class SeamlessM4TModelWithSpeechInputTest(ModelTesterMixin, unittest.TestCase):
if is_torch_available()
else ()
)
# Doesn't run generation tests. Custom generation method with a different interface
all_generative_model_classes = ()
def setUp(self):
self.model_tester = SeamlessM4TModelTester(self, input_modality="speech")
@@ -580,9 +581,7 @@ class SeamlessM4TModelWithSpeechInputTest(ModelTesterMixin, unittest.TestCase):
@require_torch
class SeamlessM4TModelWithTextInputTest(
ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase
):
class SeamlessM4TModelWithTextInputTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
is_encoder_decoder = True
fx_compatible = False
test_missing_keys = False

View File

@@ -23,7 +23,6 @@ from transformers.testing_utils import require_torch, slow, torch_device
from transformers.trainer_utils import set_seed
from transformers.utils import cached_property
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
ModelTesterMixin,
@@ -374,6 +373,8 @@ class SeamlessM4Tv2ModelWithSpeechInputTest(ModelTesterMixin, unittest.TestCase)
if is_torch_available()
else ()
)
# Doesn't run generation tests. Has custom generation method with a different interface
all_generative_model_classes = ()
def setUp(self):
self.model_tester = SeamlessM4Tv2ModelTester(self, input_modality="speech")
@@ -595,7 +596,7 @@ class SeamlessM4Tv2ModelWithSpeechInputTest(ModelTesterMixin, unittest.TestCase)
@require_torch
class SeamlessM4Tv2ModelWithTextInputTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
class SeamlessM4Tv2ModelWithTextInputTest(ModelTesterMixin, unittest.TestCase):
is_encoder_decoder = True
fx_compatible = False
test_missing_keys = False

View File

@@ -362,6 +362,8 @@ class SpeechT5ForSpeechToTextTester:
@require_torch
class SpeechT5ForSpeechToTextTest(ModelTesterMixin, unittest.TestCase):
all_model_classes = (SpeechT5ForSpeechToText,) if is_torch_available() else ()
# Doesn't run generation tests. TODO eustache/joao: shape checks probably need an update
all_generative_model_classes = ()
is_encoder_decoder = True
test_pruning = False
test_headmasking = False

View File

@@ -23,7 +23,6 @@ from huggingface_hub import hf_hub_download
from transformers import ResNetConfig, TableTransformerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -189,7 +188,7 @@ class TableTransformerModelTester:
@require_torch
class TableTransformerModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
class TableTransformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (
(
TableTransformerModel,

View File

@@ -30,6 +30,7 @@ from transformers.testing_utils import (
)
from transformers.utils import cached_property
from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin
@@ -265,7 +266,7 @@ class UdopModelTester:
@require_torch
class UdopModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
class UdopModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
all_model_classes = (
(
UdopModel,
@@ -419,6 +420,14 @@ class UdopModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
model = UdopForConditionalGeneration.from_pretrained(model_name)
self.assertIsNotNone(model)
@unittest.skip(reason="TODO: Fix me @joao")
def test_generate_with_head_masking(self):
pass
@unittest.skip(reason="TODO: Fix me @joao")
def test_generate_without_input_ids(self):
pass
class UdopEncoderOnlyModelTester:
def __init__(

View File

@@ -3353,7 +3353,7 @@ class WhisperEncoderModelTester:
@require_torch
class WhisperEncoderModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
class WhisperEncoderModelTest(ModelTesterMixin, unittest.TestCase):
all_model_classes = (WhisperForAudioClassification,) if is_torch_available() else ()
is_encoder_decoder = False
fx_compatible = False