From 43d17c18360ac9c3d3491389328e2fe55fe8f9ce Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 29 Mar 2024 11:51:20 +0100 Subject: [PATCH] Mark `test_eager_matches_sdpa_generate` flaky for some models (#29479) * fix * revert for qwen2 * revert for qwen2 * update * update --------- Co-authored-by: ydshieh --- tests/models/falcon/test_modeling_falcon.py | 11 ++++++++++- tests/models/mistral/test_modeling_mistral.py | 8 ++++++++ tests/models/mixtral/test_modeling_mixtral.py | 9 +++++++++ tests/models/qwen2/test_modeling_qwen2.py | 8 ++++++++ tests/models/qwen2_moe/test_modeling_qwen2_moe.py | 6 ++++++ tests/models/stablelm/test_modeling_stablelm.py | 3 +++ 6 files changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/models/falcon/test_modeling_falcon.py b/tests/models/falcon/test_modeling_falcon.py index 17b3dc42cf..35708d868d 100644 --- a/tests/models/falcon/test_modeling_falcon.py +++ b/tests/models/falcon/test_modeling_falcon.py @@ -27,7 +27,14 @@ from transformers import ( is_torch_available, set_seed, ) -from transformers.testing_utils import require_bitsandbytes, require_torch, require_torch_sdpa, slow, torch_device +from transformers.testing_utils import ( + is_flaky, + require_bitsandbytes, + require_torch, + require_torch_sdpa, + slow, + torch_device, +) from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester @@ -503,6 +510,8 @@ class FalconModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix torch.testing.assert_close(ntk_sin_long, original_sin_long) self.assertTrue((ntk_scaling_rope.inv_freq <= original_rope.inv_freq).all()) + # TODO: @Fxmarty + @is_flaky(max_attempts=3, description="flaky on some models.") @require_torch_sdpa @slow def test_eager_matches_sdpa_generate(self): diff --git a/tests/models/mistral/test_modeling_mistral.py b/tests/models/mistral/test_modeling_mistral.py index 5e91e70ecd..2e675a2851 100644 --- a/tests/models/mistral/test_modeling_mistral.py +++ b/tests/models/mistral/test_modeling_mistral.py @@ -24,6 +24,7 @@ import pytest from transformers import AutoTokenizer, MistralConfig, is_torch_available, set_seed from transformers.testing_utils import ( backend_empty_cache, + is_flaky, require_bitsandbytes, require_flash_attn, require_torch, @@ -309,6 +310,13 @@ class MistralModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMi ): return True + # TODO: @Fxmarty + @is_flaky(max_attempts=3, description="flaky on some models.") + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_generate(self): + super().test_eager_matches_sdpa_generate() + def setUp(self): self.model_tester = MistralModelTester(self) self.config_tester = ConfigTester(self, config_class=MistralConfig, hidden_size=37) diff --git a/tests/models/mixtral/test_modeling_mixtral.py b/tests/models/mixtral/test_modeling_mixtral.py index b6dcee093e..efd48d6a9c 100644 --- a/tests/models/mixtral/test_modeling_mixtral.py +++ b/tests/models/mixtral/test_modeling_mixtral.py @@ -22,9 +22,11 @@ import pytest from transformers import MixtralConfig, is_torch_available from transformers.testing_utils import ( + is_flaky, require_flash_attn, require_torch, require_torch_gpu, + require_torch_sdpa, slow, torch_device, ) @@ -307,6 +309,13 @@ class MixtralModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMi ): return True + # TODO: @Fxmarty + @is_flaky(max_attempts=3, description="flaky on some models.") + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_generate(self): + super().test_eager_matches_sdpa_generate() + def setUp(self): self.model_tester = MixtralModelTester(self) self.config_tester = ConfigTester(self, config_class=MixtralConfig, hidden_size=37) diff --git a/tests/models/qwen2/test_modeling_qwen2.py b/tests/models/qwen2/test_modeling_qwen2.py index 587312bfa2..49da4fec98 100644 --- a/tests/models/qwen2/test_modeling_qwen2.py +++ b/tests/models/qwen2/test_modeling_qwen2.py @@ -320,6 +320,14 @@ class Qwen2ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixi ): return True + # Ignore copy + # TODO: @Fxmarty + @require_torch_sdpa + @slow + @unittest.skip(reason="Currently failing.") + def test_eager_matches_sdpa_generate(self): + super().test_eager_matches_sdpa_generate() + def setUp(self): self.model_tester = Qwen2ModelTester(self) self.config_tester = ConfigTester(self, config_class=Qwen2Config, hidden_size=37) diff --git a/tests/models/qwen2_moe/test_modeling_qwen2_moe.py b/tests/models/qwen2_moe/test_modeling_qwen2_moe.py index 808f8b3a87..b6f8df27ef 100644 --- a/tests/models/qwen2_moe/test_modeling_qwen2_moe.py +++ b/tests/models/qwen2_moe/test_modeling_qwen2_moe.py @@ -349,6 +349,12 @@ class Qwen2MoeModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterM ): return True + # Ignore copy + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_generate(self): + super().test_eager_matches_sdpa_generate() + def setUp(self): self.model_tester = Qwen2MoeModelTester(self) self.config_tester = ConfigTester(self, config_class=Qwen2MoeConfig, hidden_size=37) diff --git a/tests/models/stablelm/test_modeling_stablelm.py b/tests/models/stablelm/test_modeling_stablelm.py index 64f828825c..0f01fa8fca 100644 --- a/tests/models/stablelm/test_modeling_stablelm.py +++ b/tests/models/stablelm/test_modeling_stablelm.py @@ -21,6 +21,7 @@ from parameterized import parameterized from transformers import StableLmConfig, is_torch_available, set_seed from transformers.testing_utils import ( + is_flaky, require_bitsandbytes, require_flash_attn, require_torch, @@ -500,6 +501,8 @@ class StableLmModelIntegrationTest(unittest.TestCase): self.assertEqual(EXPECTED_OUTPUT_TOKEN_IDS, generated_ids[0][-3:].tolist()) # Copied from transformers.tests.models.llama.test_modeling_llama.LlamaModelTest.test_eager_matches_sdpa_generate with Llama->StableLm,saibo/llama-1B->stabilityai/stablelm-3b-4e1t + # TODO: @Fxmarty + @is_flaky(max_attempts=3, description="flaky on some models.") @require_torch_sdpa @slow def test_eager_matches_sdpa_generate(self):