From aea7b23b57ea0bbe15c42114d35b3c7a4d79d7eb Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Thu, 18 May 2023 11:04:51 +0100 Subject: [PATCH] Generate: skip left-padding tests on old models (#23437) --- tests/generation/test_utils.py | 4 +--- tests/models/bart/test_modeling_bart.py | 4 ++++ .../bigbird_pegasus/test_modeling_bigbird_pegasus.py | 4 ++++ tests/models/blenderbot/test_modeling_blenderbot.py | 4 ++++ .../blenderbot_small/test_modeling_blenderbot_small.py | 4 ++++ tests/models/ctrl/test_modeling_ctrl.py | 4 ++++ tests/models/imagegpt/test_modeling_imagegpt.py | 4 ++++ tests/models/marian/test_modeling_marian.py | 4 ++++ tests/models/mbart/test_modeling_mbart.py | 4 ++++ tests/models/mvp/test_modeling_mvp.py | 4 ++++ tests/models/pegasus/test_modeling_pegasus.py | 4 ++++ tests/models/plbart/test_modeling_plbart.py | 4 ++++ tests/models/prophetnet/test_modeling_prophetnet.py | 4 ++++ tests/models/reformer/test_modeling_reformer.py | 8 ++++++++ tests/models/transfo_xl/test_modeling_transfo_xl.py | 4 ++++ tests/models/trocr/test_modeling_trocr.py | 4 ++++ 16 files changed, 65 insertions(+), 3 deletions(-) diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index 06e56498d9..96649b953c 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -1597,9 +1597,7 @@ class GenerationTesterMixin: attn_weights = out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1] self.assertEqual(sum([w.sum().item() for w in attn_weights]), 0.0) - # TODO (joao): this test is actually not slow :) However, it is not passing in some models (e.g. GPTNeoX) and the - # fix for some models is quite lengthy. Being slow means it doesn't block our push CI while we fix it. - @slow + @slow # TODO (Joao): fix GPTBigCode def test_left_padding_compatibility(self): # The check done in this test is fairly difficult -- depending on the model architecture, passing the right # position index for the position embeddings can still result in a different output, due to numerical masking. diff --git a/tests/models/bart/test_modeling_bart.py b/tests/models/bart/test_modeling_bart.py index a4b77e8431..949e647e6f 100644 --- a/tests/models/bart/test_modeling_bart.py +++ b/tests/models/bart/test_modeling_bart.py @@ -1516,3 +1516,7 @@ class BartStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, un def test_save_load_fast_init_from_base(self): pass + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py b/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py index 32b7bbcbb2..5d345db3fc 100644 --- a/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py +++ b/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py @@ -811,3 +811,7 @@ class BigBirdPegasusStandaloneDecoderModelTest(ModelTesterMixin, GenerationTeste def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/blenderbot/test_modeling_blenderbot.py b/tests/models/blenderbot/test_modeling_blenderbot.py index 8762d11aaa..499c7aa521 100644 --- a/tests/models/blenderbot/test_modeling_blenderbot.py +++ b/tests/models/blenderbot/test_modeling_blenderbot.py @@ -561,3 +561,7 @@ class BlenderbotStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMix def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/blenderbot_small/test_modeling_blenderbot_small.py b/tests/models/blenderbot_small/test_modeling_blenderbot_small.py index f8d247ba12..257aa1699c 100644 --- a/tests/models/blenderbot_small/test_modeling_blenderbot_small.py +++ b/tests/models/blenderbot_small/test_modeling_blenderbot_small.py @@ -566,3 +566,7 @@ class BlenderbotSmallStandaloneDecoderModelTest(ModelTesterMixin, GenerationTest def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/ctrl/test_modeling_ctrl.py b/tests/models/ctrl/test_modeling_ctrl.py index 87d719853a..8941927f17 100644 --- a/tests/models/ctrl/test_modeling_ctrl.py +++ b/tests/models/ctrl/test_modeling_ctrl.py @@ -249,6 +249,10 @@ class CTRLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin model = CTRLModel.from_pretrained(model_name) self.assertIsNotNone(model) + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass + @require_torch class CTRLModelLanguageGenerationTest(unittest.TestCase): diff --git a/tests/models/imagegpt/test_modeling_imagegpt.py b/tests/models/imagegpt/test_modeling_imagegpt.py index e6c8524c6d..27d83f3eb8 100644 --- a/tests/models/imagegpt/test_modeling_imagegpt.py +++ b/tests/models/imagegpt/test_modeling_imagegpt.py @@ -520,6 +520,10 @@ class ImageGPTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterM self.assertTrue(models_equal) + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/marian/test_modeling_marian.py b/tests/models/marian/test_modeling_marian.py index fe5f860652..877f5273c9 100644 --- a/tests/models/marian/test_modeling_marian.py +++ b/tests/models/marian/test_modeling_marian.py @@ -862,3 +862,7 @@ class MarianStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/mbart/test_modeling_mbart.py b/tests/models/mbart/test_modeling_mbart.py index d7094ca6e4..ec3d36f33d 100644 --- a/tests/models/mbart/test_modeling_mbart.py +++ b/tests/models/mbart/test_modeling_mbart.py @@ -692,3 +692,7 @@ class MBartStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, u def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/mvp/test_modeling_mvp.py b/tests/models/mvp/test_modeling_mvp.py index e996a998a8..cc3986a370 100644 --- a/tests/models/mvp/test_modeling_mvp.py +++ b/tests/models/mvp/test_modeling_mvp.py @@ -818,3 +818,7 @@ class MvpStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, uni def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/pegasus/test_modeling_pegasus.py b/tests/models/pegasus/test_modeling_pegasus.py index cb8b36c9af..bde7477f94 100644 --- a/tests/models/pegasus/test_modeling_pegasus.py +++ b/tests/models/pegasus/test_modeling_pegasus.py @@ -573,3 +573,7 @@ class PegasusStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/plbart/test_modeling_plbart.py b/tests/models/plbart/test_modeling_plbart.py index 6ad226747a..05dbac6a2c 100644 --- a/tests/models/plbart/test_modeling_plbart.py +++ b/tests/models/plbart/test_modeling_plbart.py @@ -658,3 +658,7 @@ class PLBartStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass diff --git a/tests/models/prophetnet/test_modeling_prophetnet.py b/tests/models/prophetnet/test_modeling_prophetnet.py index baf7351bf5..fa717b2743 100644 --- a/tests/models/prophetnet/test_modeling_prophetnet.py +++ b/tests/models/prophetnet/test_modeling_prophetnet.py @@ -1146,6 +1146,10 @@ class ProphetNetStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMix # decoder cannot keep gradients return + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass + @require_torch class ProphetNetStandaloneEncoderModelTest(ModelTesterMixin, unittest.TestCase): diff --git a/tests/models/reformer/test_modeling_reformer.py b/tests/models/reformer/test_modeling_reformer.py index 39e1389477..c84f729633 100644 --- a/tests/models/reformer/test_modeling_reformer.py +++ b/tests/models/reformer/test_modeling_reformer.py @@ -682,6 +682,10 @@ class ReformerLocalAttnModelTest(ReformerTesterMixin, GenerationTesterMixin, Mod [expected_shape] * len(iter_hidden_states), ) + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass + @require_torch class ReformerLSHAttnModelTest( @@ -839,6 +843,10 @@ class ReformerLSHAttnModelTest( def test_past_key_values_format(self): pass + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass + @require_torch @require_sentencepiece diff --git a/tests/models/transfo_xl/test_modeling_transfo_xl.py b/tests/models/transfo_xl/test_modeling_transfo_xl.py index c86cd704f1..970f87bf10 100644 --- a/tests/models/transfo_xl/test_modeling_transfo_xl.py +++ b/tests/models/transfo_xl/test_modeling_transfo_xl.py @@ -486,6 +486,10 @@ class TransfoXLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester weight = getattr(module, param) weight.data.fill_(3) + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass + @require_torch class TransfoXLModelLanguageGenerationTest(unittest.TestCase): diff --git a/tests/models/trocr/test_modeling_trocr.py b/tests/models/trocr/test_modeling_trocr.py index f670f4e6d9..0033f339ae 100644 --- a/tests/models/trocr/test_modeling_trocr.py +++ b/tests/models/trocr/test_modeling_trocr.py @@ -192,3 +192,7 @@ class TrOCRStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, P # decoder cannot keep gradients def test_retain_grad_hidden_states_attentions(self): return + + @unittest.skip("The model doesn't support left padding") # and it's not used enough to be worth fixing :) + def test_left_padding_compatibility(self): + pass