From f2d5dfbab2af25d6fbfb4a315c78bcdfad9f62d7 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:10:42 +0100 Subject: [PATCH] Remove `@slow` for `test_eager_matches_sdpa_inference` (#34558) * update * update * update * update * update * update * update * update * update * update * update --------- Co-authored-by: ydshieh --- .../modeling_llava_next_video.py | 2 +- .../modular_llava_next_video.py | 2 +- .../models/unispeech/modeling_unispeech.py | 2 +- tests/generation/test_utils.py | 20 +- tests/models/albert/test_modeling_albert.py | 9 +- tests/models/glm/test_modeling_glm.py | 302 ------------------ tests/models/granite/test_modeling_granite.py | 10 - .../granitemoe/test_modeling_granitemoe.py | 10 - tests/models/idefics/test_modeling_idefics.py | 15 +- .../llava_next/test_modeling_llava_next.py | 8 +- .../test_modeling_llava_next_video.py | 10 +- tests/models/mimi/test_modeling_mimi.py | 78 ++--- tests/models/mllama/test_modeling_mllama.py | 9 - .../models/musicgen/test_modeling_musicgen.py | 180 +++++------ .../test_modeling_musicgen_melody.py | 154 ++++----- .../models/nemotron/test_modeling_nemotron.py | 10 - .../paligemma/test_modeling_paligemma.py | 10 - .../models/qwen2_vl/test_modeling_qwen2_vl.py | 10 +- .../video_llava/test_modeling_video_llava.py | 13 +- .../models/videomae/test_modeling_videomae.py | 7 +- tests/test_modeling_common.py | 36 ++- 21 files changed, 271 insertions(+), 626 deletions(-) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 73118f4bfc..b0a20d6c5c 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -95,7 +95,7 @@ class LlavaNextVideoPooler(nn.Module): mode = config.spatial_pool_mode stride = config.spatial_pool_stride out_channels = getattr(config, "spatial_pool_out_channels", config.vision_config.hidden_size) - self.image_size = config.vision_config.image_size // config.vision_config.patch_size**2 + self.image_size = (config.vision_config.image_size // config.vision_config.patch_size) ** 2 if mode == "average": self.pool = nn.AvgPool2d(kernel_size=stride, stride=stride) diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index 002b450c2a..3d6431d7ea 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -191,7 +191,7 @@ class LlavaNextVideoPooler(nn.Module): mode = config.spatial_pool_mode stride = config.spatial_pool_stride out_channels = getattr(config, "spatial_pool_out_channels", config.vision_config.hidden_size) - self.image_size = config.vision_config.image_size // config.vision_config.patch_size**2 + self.image_size = (config.vision_config.image_size // config.vision_config.patch_size) ** 2 if mode == "average": self.pool = nn.AvgPool2d(kernel_size=stride, stride=stride) diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index 52ba08f5d4..6ce5e77706 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -1577,7 +1577,7 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): quantized_features, codevector_perplexity = self.quantizer(extract_features) # project quantized features twice - quantized_features = self.project_q(quantized_features) + quantized_features = self.project_q(quantized_features.to(self.project_q.weight.dtype)) quantized_features = self.project_hid(quantized_features) prob_replace_matrix = torch.empty(transformer_features.size(0), transformer_features.size(1)).fill_( diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index 3bd8ce4b59..cbe851e97e 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -185,16 +185,16 @@ class GenerationTesterMixin: # This is a band-aid for VLM models, to ensure they don't generate image/video tokens which would cause them # to crash. On pretrained models this isn't a risk, as they are trained to not generate these tokens. if config is not None: - image_token_index = ( - config.image_token_index - if getattr(config, "image_token_index", None) is not None - else getattr(config, "image_token_id", None) - ) - video_token_index = config.video_token_index if hasattr(config, "video_token_index") else None - if image_token_index is not None and image_token_index < config.get_text_config().vocab_size: - logits_processor_kwargs["bad_words_ids"].append([image_token_index]) - if video_token_index is not None and video_token_index < config.get_text_config().vocab_size: - logits_processor_kwargs["bad_words_ids"].append([video_token_index]) + for key in [ + "image_token_index", + "image_token_id", + "video_token_index", + "video_token_id", + "vision_start_token_id", + ]: + token_index = getattr(config, key, None) + if token_index is not None and token_index < config.get_text_config().vocab_size: + logits_processor_kwargs["bad_words_ids"].append([token_index]) return logits_processor_kwargs diff --git a/tests/models/albert/test_modeling_albert.py b/tests/models/albert/test_modeling_albert.py index 970f1dd855..0a123c02ab 100644 --- a/tests/models/albert/test_modeling_albert.py +++ b/tests/models/albert/test_modeling_albert.py @@ -17,10 +17,11 @@ import unittest from packaging import version +from parameterized import parameterized from transformers import AlbertConfig, AutoTokenizer, is_torch_available from transformers.models.auto import get_values -from transformers.testing_utils import require_torch, slow, torch_device +from transformers.testing_utils import require_torch, require_torch_sdpa, slow, torch_device from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask @@ -288,6 +289,12 @@ class AlbertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): self.model_tester = AlbertModelTester(self) self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @unittest.skip("Albert requires `head_mask` which is currently not done in this test.") + def test_eager_matches_sdpa_inference(self): + pass + def test_config(self): self.config_tester.run_common_tests() diff --git a/tests/models/glm/test_modeling_glm.py b/tests/models/glm/test_modeling_glm.py index b92c5db815..ebac3b9167 100644 --- a/tests/models/glm/test_modeling_glm.py +++ b/tests/models/glm/test_modeling_glm.py @@ -14,13 +14,9 @@ # limitations under the License. """Testing suite for the PyTorch Glm model.""" -import inspect -import tempfile import unittest -import numpy as np import pytest -from parameterized import parameterized from transformers import AutoModelForCausalLM, AutoTokenizer, GlmConfig, is_torch_available from transformers.testing_utils import ( @@ -32,7 +28,6 @@ from transformers.testing_utils import ( slow, torch_device, ) -from transformers.utils import is_torch_bf16_available_on_device, is_torch_fp16_available_on_device from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester @@ -421,303 +416,6 @@ class GlmModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, torch.testing.assert_close(normalized_0, normalized_1, rtol=1e-3, atol=1e-3) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @require_torch_sdpa - @slow - @is_flaky - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - """Overwrite to add flakyness: some cases can sometimes fail""" - if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): - self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") - - if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): - self.skipTest( - f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" - ) - - # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. - if torch_dtype == "float16": - torch_dtype = torch.float16 - elif torch_dtype == "bfloat16": - torch_dtype = torch.bfloat16 - elif torch_dtype == "float32": - torch_dtype = torch.float32 - - atols = { - ("cpu", False, torch.float32): 1e-6, - ("cpu", False, torch.bfloat16): 1e-2, - ("cpu", True, torch.float32): 1e-6, - ("cpu", True, torch.bfloat16): 1e-2, - ("cuda", False, torch.float32): 1e-6, - ("cuda", False, torch.bfloat16): 1e-2, - ("cuda", False, torch.float16): 5e-3, - ("cuda", True, torch.float32): 1e-6, - ("cuda", True, torch.bfloat16): 1e-2, - ("cuda", True, torch.float16): 5e-3, - } - rtols = { - ("cpu", False, torch.float32): 1e-4, - ("cpu", False, torch.bfloat16): 1e-2, - ("cpu", True, torch.float32): 1e-4, - ("cpu", True, torch.bfloat16): 1e-2, - ("cuda", False, torch.float32): 1e-4, - ("cuda", False, torch.bfloat16): 1e-2, - ("cuda", False, torch.float16): 5e-3, - ("cuda", True, torch.float32): 1e-4, - ("cuda", True, torch.bfloat16): 3e-2, - ("cuda", True, torch.float16): 5e-3, - } - - def get_mean_reldiff(failcase, x, ref, atol, rtol): - return f"{failcase}: mean relative difference: {((x - ref).abs() / (ref.abs() + 1e-12)).mean():.3e}, torch atol = {atol}, torch rtol = {rtol}" - - for model_class in self.all_model_classes: - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - model = model_class(config) - # FIXME: we deactivate boolean mask for models using "use_mask_token" in their constructors. - # These models support masking only in the case `use_mask_token=True`. Otherwise they cannot consume an input mask. - # This means that the class needs to be instantiated much later, after `use_mask` is set, which means a significant refactor of the code. - # However masking there is not done at any layers that matters (i.e self-attention), therefore we can safely deactivate it. - deactivate_mask = "use_mask_token" in inspect.signature(model_class).parameters - - is_encoder_decoder = model.config.is_encoder_decoder - - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) - model_sdpa = model_sdpa.eval().to(torch_device) - - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - - model_eager = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch_dtype, - attn_implementation="eager", - ) - model_eager = model_eager.eval().to(torch_device) - - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 16 times the model, - # but it would be nicer to have an efficient way to use parameterized.expand - fail_cases = [] - for padding_side in ["left", "right"]: - for use_mask in [False, True]: - for output_attentions in [True, False]: - can_output_attn = "output_attentions" in inspect.signature(model_sdpa.forward).parameters - if not (self.has_attentions and can_output_attn) and output_attentions: - continue - for batch_size in [1, 5]: - dummy_input = inputs_dict[model.main_input_name] - - if dummy_input.dtype in [torch.float32, torch.bfloat16, torch.float16]: - dummy_input = dummy_input.to(torch_dtype) - - dummy_input = dummy_input[:batch_size] - if dummy_input.shape[0] != batch_size: - if dummy_input.dtype in [torch.float32, torch.bfloat16, torch.float16]: - extension = torch.rand( - batch_size - dummy_input.shape[0], - *dummy_input.shape[1:], - dtype=torch_dtype, - device=torch_device, - ) - dummy_input = torch.cat((dummy_input, extension), dim=0).to(torch_device) - else: - extension = torch.randint( - high=5, - size=(batch_size - dummy_input.shape[0], *dummy_input.shape[1:]), - dtype=dummy_input.dtype, - device=torch_device, - ) - dummy_input = torch.cat((dummy_input, extension), dim=0).to(torch_device) - - if not use_mask: - dummy_attention_mask = None - else: - dummy_attention_mask = inputs_dict.get("attention_mask", None) - if dummy_attention_mask is None: - if is_encoder_decoder: - seqlen = inputs_dict.get("decoder_input_ids", dummy_input).shape[-1] - else: - seqlen = dummy_input.shape[-1] - dummy_attention_mask = ( - torch.ones(batch_size, seqlen).to(torch.int64).to(torch_device) - ) - - dummy_attention_mask = dummy_attention_mask[:batch_size] - if dummy_attention_mask.shape[0] != batch_size: - extension = torch.ones( - batch_size - dummy_attention_mask.shape[0], - *dummy_attention_mask.shape[1:], - dtype=dummy_attention_mask.dtype, - device=torch_device, - ) - dummy_attention_mask = torch.cat((dummy_attention_mask, extension), dim=0) - dummy_attention_mask = dummy_attention_mask.to(torch_device) - - dummy_attention_mask[:] = 1 - if padding_side == "left": - dummy_attention_mask[-1, :-1] = 1 - dummy_attention_mask[-1, -4:] = 0 - elif padding_side == "right": - dummy_attention_mask[-1, 1:] = 1 - dummy_attention_mask[-1, :3] = 0 - - for enable_kernels in [False, True]: - failcase = f"padding_side={padding_side}, use_mask={use_mask}, batch_size={batch_size}, enable_kernels={enable_kernels}" - if is_encoder_decoder: - decoder_input_ids = inputs_dict.get("decoder_input_ids", dummy_input)[ - :batch_size - ] - if decoder_input_ids.shape[0] != batch_size: - extension = torch.ones( - batch_size - decoder_input_ids.shape[0], - *decoder_input_ids.shape[1:], - dtype=decoder_input_ids.dtype, - device=torch_device, - ) - decoder_input_ids = torch.cat((decoder_input_ids, extension), dim=0) - decoder_input_ids = decoder_input_ids.to(torch_device) - - # TODO: never an `attention_mask` arg here? - processed_inputs = { - model.main_input_name: dummy_input, - "decoder_input_ids": decoder_input_ids, - "decoder_attention_mask": dummy_attention_mask, - "output_hidden_states": True, - } - else: - processed_inputs = { - model.main_input_name: dummy_input, - "output_hidden_states": True, - } - - # Otherwise fails for e.g. WhisperEncoderModel - if "attention_mask" in inspect.signature(model_eager.forward).parameters: - processed_inputs["attention_mask"] = dummy_attention_mask - - if ( - self.has_attentions - and "output_attentions" in inspect.signature(model_sdpa.forward).parameters - ): - processed_inputs["output_attentions"] = output_attentions - if not deactivate_mask and ( - "bool_masked_pos" in inspect.signature(model_eager.forward).parameters - ): - dummy_mask = torch.ones((self.model_tester.num_masks,)) - - # In case of additional token (like class) we define a custom `mask_length` - if hasattr(self.model_tester, "mask_length"): - mask_length = self.model_tester.mask_length - dummy_mask.size(0) - else: - mask_length = self.model_tester.seq_length - dummy_mask.size(0) - dummy_mask = torch.cat([dummy_mask, torch.zeros(mask_length)]) - dummy_bool_masked_pos = dummy_mask.expand(batch_size, -1).bool() - processed_inputs["bool_masked_pos"] = dummy_bool_masked_pos.to(torch_device) - - if "noise" in inspect.signature(model_eager.forward).parameters: - np.random.seed(2) - num_patches = int( - (self.model_tester.image_size // self.model_tester.patch_size) ** 2 - ) - noise = np.random.uniform(size=(batch_size, num_patches)) - processed_inputs["noise"] = torch.from_numpy(noise) - - # TODO: test gradients as well (& for FA2 as well!) - with torch.no_grad(): - with torch.backends.cuda.sdp_kernel( - enable_flash=enable_kernels, - enable_math=True, - enable_mem_efficient=enable_kernels, - ): - prepared_inputs = self._prepare_for_class(processed_inputs, model_class) - outputs_eager = model_eager(**prepared_inputs) - outputs_sdpa = model_sdpa(**prepared_inputs) - - logits_eager = ( - outputs_eager.hidden_states[-1] - if not is_encoder_decoder - else outputs_eager.decoder_hidden_states[-1] - ) - logits_sdpa = ( - outputs_sdpa.hidden_states[-1] - if not is_encoder_decoder - else outputs_sdpa.decoder_hidden_states[-1] - ) - - if torch_device in ["cpu", "cuda"]: - atol = atols[torch_device, enable_kernels, torch_dtype] - rtol = rtols[torch_device, enable_kernels, torch_dtype] - else: - atol = 1e-7 - rtol = 1e-4 - - # Masked tokens output slightly deviates - we don't mind that. - if use_mask: - if padding_side == "left": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - sub_sdpa = logits_sdpa[-1, :-4] - sub_eager = logits_eager[-1, :-4] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, -4:] - # sub_eager = logits_eager[-1, -4:] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - elif padding_side == "right": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - sub_sdpa = logits_sdpa[-1, 3:] - sub_eager = logits_eager[-1, 3:] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, :3] - # sub_eager = logits_eager[-1, :3] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - - else: - if not torch.allclose(logits_sdpa, logits_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) - ) - - self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) - @slow @require_torch_accelerator diff --git a/tests/models/granite/test_modeling_granite.py b/tests/models/granite/test_modeling_granite.py index 97b59f5aa5..60eb964927 100644 --- a/tests/models/granite/test_modeling_granite.py +++ b/tests/models/granite/test_modeling_granite.py @@ -25,7 +25,6 @@ from transformers.testing_utils import ( require_read_token, require_torch, require_torch_gpu, - require_torch_sdpa, slow, torch_device, ) @@ -445,15 +444,6 @@ class GraniteModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMi if not has_flash: raise ValueError("The flash model should have flash attention layers") - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - """ - skipping the test since mup is very flaky and gets consistently different outputs - """ - self.skipTest("skipping the test since mup is very flaky and gets consistently different outputs") - @require_torch_gpu class GraniteIntegrationTest(unittest.TestCase): diff --git a/tests/models/granitemoe/test_modeling_granitemoe.py b/tests/models/granitemoe/test_modeling_granitemoe.py index f2f76b9fa7..97af65667e 100644 --- a/tests/models/granitemoe/test_modeling_granitemoe.py +++ b/tests/models/granitemoe/test_modeling_granitemoe.py @@ -25,7 +25,6 @@ from transformers.testing_utils import ( require_read_token, require_torch, require_torch_gpu, - require_torch_sdpa, slow, torch_device, ) @@ -444,15 +443,6 @@ class GraniteMoeModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.Test if not has_flash: raise ValueError("The flash model should have flash attention layers") - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - """ - skipping the test since mup is very flaky and gets consistently different outputs - """ - self.skipTest("skipping the test since mup is very flaky and gets consistently different outputs") - @require_torch_gpu class GraniteMoeIntegrationTest(unittest.TestCase): diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index 7be87fd783..12004cc3c8 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -134,7 +134,7 @@ class IdeficsModelTester: num_attention_heads=self.vision_num_attention_heads, num_hidden_layers=self.vision_num_hidden_layers, intermediate_size=self.vision_intermediate_size, - ) + ).to_dict() self.perceiver_qk_layer_norms_perceiver = perceiver_qk_layer_norms_perceiver self.perceiver_resampler_depth = perceiver_resampler_depth @@ -316,7 +316,6 @@ class IdeficsModelTester: return floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) @require_torch_sdpa - @slow @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) def test_eager_matches_sdpa_inference(self, torch_dtype: str): self.skipTest(reason="Idefics has a hard requirement on SDPA, skipping this test") @@ -353,6 +352,12 @@ class IdeficsModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) return inputs_dict + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @unittest.skip("Idefics requires both text and image inputs which is currently not done in this test.") + def test_eager_matches_sdpa_inference(self): + pass + def test_model_outputs_equivalence(self): try: orig = self.all_model_classes @@ -602,6 +607,12 @@ class IdeficsForVisionText2TextTest(IdeficsModelTest, GenerationTesterMixin, uni ) self.config_tester = ConfigTester(self, config_class=IdeficsConfig, hidden_size=37) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @unittest.skip("Idefics requires both text and image inputs which is currently not done in this test.") + def test_eager_matches_sdpa_inference(self, torch_dtype): + pass + @pytest.mark.generate def test_left_padding_compatibility(self): """Overwrite because IDEFICS needs image attention mask to be also padded""" diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 7ce57dcba3..82508f57e0 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -90,7 +90,7 @@ class LlavaNextVisionText2TextModelTester: }, is_training=True, vision_config={ - "image_size": 16, + "image_size": 8, "patch_size": 4, "num_channels": 3, "is_training": True, @@ -123,10 +123,10 @@ class LlavaNextVisionText2TextModelTester: self.batch_size = 3 self.num_channels = 3 self.image_size = 30 - self.encoder_seq_length = 95 - self.image_grid_pinpoints = [[32, 32]] - self.num_image_tokens = 88 + self.image_grid_pinpoints = [[16, 16]] + self.num_image_tokens = 24 self.seq_length = seq_length + self.num_image_tokens + self.encoder_seq_length = self.seq_length def get_config(self): return LlavaNextConfig( diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 3ebb5752bd..83caabe16b 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -91,7 +91,7 @@ class LlavaNextVideoVisionText2TextModelTester: }, is_training=True, vision_config={ - "image_size": 16, + "image_size": 8, "patch_size": 4, "num_channels": 3, "is_training": True, @@ -125,10 +125,10 @@ class LlavaNextVideoVisionText2TextModelTester: self.batch_size = 3 self.num_channels = 3 self.image_size = 30 - self.encoder_seq_length = 127 - self.image_grid_pinpoints = [[32, 32]] - self.num_image_tokens = 88 - self.num_video_tokens = 32 + + self.image_grid_pinpoints = [[16, 16]] + self.num_image_tokens = 24 + self.num_video_tokens = 8 self.seq_length = seq_length + self.num_image_tokens + self.num_video_tokens def get_config(self): diff --git a/tests/models/mimi/test_modeling_mimi.py b/tests/models/mimi/test_modeling_mimi.py index df0007d666..7ddc6b7474 100644 --- a/tests/models/mimi/test_modeling_mimi.py +++ b/tests/models/mimi/test_modeling_mimi.py @@ -409,10 +409,14 @@ class MimiModelTest(ModelTesterMixin, unittest.TestCase): config.use_conv_shortcut = False self.model_tester.create_and_check_model_forward(config, inputs_dict) + # Overwrite to use `audio_values` as the tensors to compare. + # TODO: Try to do this in the parent class. @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow def test_eager_matches_sdpa_inference(self, torch_dtype: str): + if torch_dtype == "float16" and torch_device == "cpu": + self.skipTest("`replication_pad1d` not implemented for 'Half") + if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") @@ -513,7 +517,7 @@ class MimiModelTest(ModelTesterMixin, unittest.TestCase): can_output_attn = "output_attentions" in inspect.signature(model_sdpa.forward).parameters if not (self.has_attentions and can_output_attn) and output_attentions: continue - for batch_size in [1, 5]: + for batch_size in [7]: dummy_input = inputs_dict[model.main_input_name] if dummy_input.dtype in [torch.float32, torch.bfloat16, torch.float16]: @@ -564,11 +568,11 @@ class MimiModelTest(ModelTesterMixin, unittest.TestCase): dummy_attention_mask[:] = 1 if padding_side == "left": - dummy_attention_mask[-1, :-1] = 1 - dummy_attention_mask[-1, -4:] = 0 + dummy_attention_mask[-1, :2] = 0 + dummy_attention_mask[-1, 2:] = 1 elif padding_side == "right": - dummy_attention_mask[-1, 1:] = 1 - dummy_attention_mask[-1, :3] = 0 + dummy_attention_mask[-1, -2:] = 0 + dummy_attention_mask[-1, :-2] = 1 for enable_kernels in [False, True]: failcase = f"padding_side={padding_side}, use_mask={use_mask}, batch_size={batch_size}, enable_kernels={enable_kernels}" @@ -655,52 +659,32 @@ class MimiModelTest(ModelTesterMixin, unittest.TestCase): # Masked tokens output slightly deviates - we don't mind that. if use_mask: + _logits_sdpa = torch.zeros_like(input=logits_sdpa) + _logits_eager = torch.zeros_like(input=logits_eager) + + _logits_sdpa[:-1] = logits_sdpa[:-1] + _logits_eager[:-1] = logits_eager[:-1] + if padding_side == "left": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, 2:] + _logits_eager[-1:, 2:] = logits_eager[-1:, 2:] - sub_sdpa = logits_sdpa[-1, :-4] - sub_eager = logits_eager[-1, :-4] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, -4:] - # sub_eager = logits_eager[-1, -4:] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) elif padding_side == "right": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, :-2] + _logits_eager[-1:, 2:] = logits_eager[-1:, :-2] - sub_sdpa = logits_sdpa[-1, 3:] - sub_eager = logits_eager[-1, 3:] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + logits_sdpa = _logits_sdpa + logits_eager = _logits_eager - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, :3] - # sub_eager = logits_eager[-1, :3] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - - else: - if not torch.allclose(logits_sdpa, logits_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) - ) + results = [ + torch.allclose(_logits_sdpa, _logits_eager, atol=atol, rtol=rtol) + for (_logits_sdpa, _logits_eager) in zip(logits_sdpa, logits_eager) + ] + # If 80% batch elements have matched results, it's fine + if np.mean(results) < 0.8: + fail_cases.append( + get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) + ) self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) diff --git a/tests/models/mllama/test_modeling_mllama.py b/tests/models/mllama/test_modeling_mllama.py index 9ed5d67822..8da927f815 100644 --- a/tests/models/mllama/test_modeling_mllama.py +++ b/tests/models/mllama/test_modeling_mllama.py @@ -30,12 +30,10 @@ from transformers import ( from transformers.models.mllama.configuration_mllama import MllamaTextConfig from transformers.testing_utils import ( cleanup, - is_flaky, require_bitsandbytes, require_read_token, require_torch, require_torch_gpu, - require_torch_sdpa, slow, torch_device, ) @@ -359,13 +357,6 @@ class MllamaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTester self.assertListEqual([layer_attention.shape for layer_attention in iter_attentions], expected_shapes) - @require_torch_sdpa - @slow - @is_flaky() - def test_eager_matches_sdpa_inference_1_bfloat16(self): - # A workaround to override parametrized test with flaky decorator - super().test_eager_matches_sdpa_inference_1_bfloat16() - @unittest.skip("For some unknown reasons the tests fails in CrossAttention layer when doing torch.sdpa(). ") def test_sdpa_can_compile_dynamic(self): pass diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 963cace28d..37b5af3ae7 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -452,7 +452,6 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: @@ -479,8 +478,10 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste atols = { ("cpu", False, torch.float32): 1e-6, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-6, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-6, ("cuda", False, torch.bfloat16): 1e-2, @@ -491,8 +492,10 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste } rtols = { ("cpu", False, torch.float32): 1e-4, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-4, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-4, ("cuda", False, torch.bfloat16): 1e-2, @@ -528,7 +531,7 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste fail_cases = [] for padding_side in ["left", "right"]: for use_mask in [False, True]: - for batch_size in [1, 5]: + for batch_size in [7]: # Ignore copy batch_size_input_ids = self.model_tester.num_codebooks * batch_size dummy_input = inputs_dict[model.main_input_name] @@ -585,11 +588,11 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste dummy_attention_mask[:] = 1 if padding_side == "left": - dummy_attention_mask[-1, :-1] = 1 - dummy_attention_mask[-1, -4:] = 0 + dummy_attention_mask[-1, :2] = 0 + dummy_attention_mask[-1, 2:] = 1 elif padding_side == "right": - dummy_attention_mask[-1, 1:] = 1 - dummy_attention_mask[-1, :3] = 0 + dummy_attention_mask[-1, -2:] = 0 + dummy_attention_mask[-1, :-2] = 1 for enable_kernels in [False, True]: failcase = f"padding_side={padding_side}, use_mask={use_mask}, batch_size={batch_size}, enable_kernels={enable_kernels}" @@ -632,52 +635,32 @@ class MusicgenDecoderTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste # Masked tokens output slightly deviates - we don't mind that. if use_mask: + _logits_sdpa = torch.zeros_like(input=logits_sdpa) + _logits_eager = torch.zeros_like(input=logits_eager) + + _logits_sdpa[:-1] = logits_sdpa[:-1] + _logits_eager[:-1] = logits_eager[:-1] + if padding_side == "left": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, 2:] + _logits_eager[-1:, 2:] = logits_eager[-1:, 2:] - sub_sdpa = logits_sdpa[-1, :-4] - sub_eager = logits_eager[-1, :-4] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, -4:] - # sub_eager = logits_eager[-1, -4:] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) elif padding_side == "right": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, :-2] + _logits_eager[-1:, 2:] = logits_eager[-1:, :-2] - sub_sdpa = logits_sdpa[-1, 3:] - sub_eager = logits_eager[-1, 3:] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + logits_sdpa = _logits_sdpa + logits_eager = _logits_eager - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, :3] - # sub_eager = logits_eager[-1, :3] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - - else: - if not torch.allclose(logits_sdpa, logits_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) - ) + results = [ + torch.allclose(_logits_sdpa, _logits_eager, atol=atol, rtol=rtol) + for (_logits_sdpa, _logits_eager) in zip(logits_sdpa, logits_eager) + ] + # If 80% batch elements have matched results, it's fine + if np.mean(results) < 0.8: + fail_cases.append( + get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) + ) self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) @@ -1496,8 +1479,6 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") @@ -1523,8 +1504,10 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, atols = { ("cpu", False, torch.float32): 1e-6, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-6, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-6, ("cuda", False, torch.bfloat16): 1e-2, @@ -1535,8 +1518,10 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, } rtols = { ("cpu", False, torch.float32): 1e-4, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-4, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-4, ("cuda", False, torch.bfloat16): 1e-2, @@ -1549,8 +1534,26 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, def get_mean_reldiff(failcase, x, ref, atol, rtol): return f"{failcase}: mean relative difference: {((x - ref).abs() / (ref.abs() + 1e-12)).mean():.3e}, torch atol = {atol}, torch rtol = {rtol}" + if hasattr(self.model_tester, "num_hidden_layers"): + self.model_tester.num_hidden_layers = 1 + for model_class in self.all_model_classes: config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + config.rms_norm_eps = 1.0 + config.layer_norm_eps = 1.0 + config.norm_eps = 1.0 + config.norm_epsilon = 1.0 + config.layer_norm_epsilon = 1.0 + + for attr in ["text_config", "vision_config", "text_encoder", "audio_encoder", "decoder"]: + if hasattr(config, attr): + getattr(config, attr).rms_norm_eps = 1.0 + getattr(config, attr).layer_norm_eps = 1.0 + getattr(config, attr).norm_eps = 1.0 + getattr(config, attr).norm_epsilon = 1.0 + getattr(config, attr).layer_norm_epsilon = 1.0 + model = model_class(config) is_encoder_decoder = model.config.is_encoder_decoder @@ -1567,12 +1570,19 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, ) model_eager = model_eager.eval().to(torch_device) + for x in model_eager.modules(): + if isinstance(x, (torch.nn.LayerNorm, torch.nn.GroupNorm)): + x.eps = 1.0 + for x in model_sdpa.modules(): + if isinstance(x, (torch.nn.LayerNorm, torch.nn.GroupNorm)): + x.eps = 1.0 + # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 8 times the model, # but it would be nicer to have an efficient way to use parameterized.expand fail_cases = [] for padding_side in ["left", "right"]: for use_mask in [False, True]: - for batch_size in [1, 5]: + for batch_size in [7]: dummy_input = inputs_dict[model.main_input_name] if dummy_input.dtype in [torch.float32, torch.bfloat16, torch.float16]: @@ -1622,11 +1632,11 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, dummy_attention_mask[:] = 1 if padding_side == "left": - dummy_attention_mask[-1, :-1] = 1 - dummy_attention_mask[-1, -4:] = 0 + dummy_attention_mask[-1, :2] = 0 + dummy_attention_mask[-1, 2:] = 1 elif padding_side == "right": - dummy_attention_mask[-1, 1:] = 1 - dummy_attention_mask[-1, :3] = 0 + dummy_attention_mask[-1, -2:] = 0 + dummy_attention_mask[-1, :-2] = 1 for enable_kernels in [False, True]: failcase = f"padding_side={padding_side}, use_mask={use_mask}, batch_size={batch_size}, enable_kernels={enable_kernels}" @@ -1687,52 +1697,32 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, # Masked tokens output slightly deviates - we don't mind that. if use_mask: + _logits_sdpa = torch.zeros_like(input=logits_sdpa) + _logits_eager = torch.zeros_like(input=logits_eager) + + _logits_sdpa[:-1] = logits_sdpa[:-1] + _logits_eager[:-1] = logits_eager[:-1] + if padding_side == "left": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, 2:] + _logits_eager[-1:, 2:] = logits_eager[-1:, 2:] - sub_sdpa = logits_sdpa[-1, :-4] - sub_eager = logits_eager[-1, :-4] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, -4:] - # sub_eager = logits_eager[-1, -4:] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) elif padding_side == "right": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, :-2] + _logits_eager[-1:, 2:] = logits_eager[-1:, :-2] - sub_sdpa = logits_sdpa[-1, 3:] - sub_eager = logits_eager[-1, 3:] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + logits_sdpa = _logits_sdpa + logits_eager = _logits_eager - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, :3] - # sub_eager = logits_eager[-1, :3] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - - else: - if not torch.allclose(logits_sdpa, logits_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) - ) + results = [ + torch.allclose(_logits_sdpa, _logits_eager, atol=atol, rtol=rtol) + for (_logits_sdpa, _logits_eager) in zip(logits_sdpa, logits_eager) + ] + # If 80% batch elements have matched results, it's fine + if np.mean(results) < 0.8: + fail_cases.append( + get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) + ) self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index 957db9f23b..de7a2745ca 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -460,7 +460,6 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: @@ -487,8 +486,10 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes atols = { ("cpu", False, torch.float32): 1e-6, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-6, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-6, ("cuda", False, torch.bfloat16): 1e-2, @@ -499,8 +500,10 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes } rtols = { ("cpu", False, torch.float32): 1e-4, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-4, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-4, ("cuda", False, torch.bfloat16): 1e-2, @@ -536,7 +539,7 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes fail_cases = [] for padding_side in ["left", "right"]: for use_mask in [False, True]: - for batch_size in [1, 5]: + for batch_size in [7]: # Ignore copy batch_size_input_ids = self.model_tester.num_codebooks * batch_size dummy_input = inputs_dict[model.main_input_name] @@ -593,11 +596,11 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes dummy_attention_mask[:] = 1 if padding_side == "left": - dummy_attention_mask[-1, :-1] = 1 - dummy_attention_mask[-1, -4:] = 0 + dummy_attention_mask[-1, :2] = 0 + dummy_attention_mask[-1, 2:] = 1 elif padding_side == "right": - dummy_attention_mask[-1, 1:] = 1 - dummy_attention_mask[-1, :3] = 0 + dummy_attention_mask[-1, -2:] = 0 + dummy_attention_mask[-1, :-2] = 1 for enable_kernels in [False, True]: failcase = f"padding_side={padding_side}, use_mask={use_mask}, batch_size={batch_size}, enable_kernels={enable_kernels}" @@ -640,52 +643,32 @@ class MusicgenMelodyDecoderTest(ModelTesterMixin, GenerationTesterMixin, unittes # Masked tokens output slightly deviates - we don't mind that. if use_mask: + _logits_sdpa = torch.zeros_like(input=logits_sdpa) + _logits_eager = torch.zeros_like(input=logits_eager) + + _logits_sdpa[:-1] = logits_sdpa[:-1] + _logits_eager[:-1] = logits_eager[:-1] + if padding_side == "left": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, 2:] + _logits_eager[-1:, 2:] = logits_eager[-1:, 2:] - sub_sdpa = logits_sdpa[-1, :-4] - sub_eager = logits_eager[-1, :-4] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, -4:] - # sub_eager = logits_eager[-1, -4:] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) elif padding_side == "right": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, :-2] + _logits_eager[-1:, 2:] = logits_eager[-1:, :-2] - sub_sdpa = logits_sdpa[-1, 3:] - sub_eager = logits_eager[-1, 3:] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + logits_sdpa = _logits_sdpa + logits_eager = _logits_eager - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, :3] - # sub_eager = logits_eager[-1, :3] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - - else: - if not torch.allclose(logits_sdpa, logits_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) - ) + results = [ + torch.allclose(_logits_sdpa, _logits_eager, atol=atol, rtol=rtol) + for (_logits_sdpa, _logits_eager) in zip(logits_sdpa, logits_eager) + ] + # If 80% batch elements have matched results, it's fine + if np.mean(results) < 0.8: + fail_cases.append( + get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) + ) self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) @@ -1486,7 +1469,6 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.all_model_classes[0]._supports_sdpa: @@ -1510,8 +1492,10 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester atols = { ("cpu", False, torch.float32): 1e-6, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-6, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-6, ("cuda", False, torch.bfloat16): 1e-2, @@ -1522,8 +1506,10 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester } rtols = { ("cpu", False, torch.float32): 1e-4, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-4, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-4, ("cuda", False, torch.bfloat16): 1e-2, @@ -1559,7 +1545,7 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester fail_cases = [] for padding_side in ["left", "right"]: for use_mask in [False, True]: - for batch_size in [1, 5]: + for batch_size in [7]: dummy_input = inputs_dict[model.main_input_name] if dummy_input.dtype in [torch.float32, torch.bfloat16, torch.float16]: @@ -1609,11 +1595,11 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester dummy_attention_mask[:] = 1 if padding_side == "left": - dummy_attention_mask[-1, :-1] = 1 - dummy_attention_mask[-1, -4:] = 0 + dummy_attention_mask[-1, :2] = 0 + dummy_attention_mask[-1, 2:] = 1 elif padding_side == "right": - dummy_attention_mask[-1, 1:] = 1 - dummy_attention_mask[-1, :3] = 0 + dummy_attention_mask[-1, -2:] = 0 + dummy_attention_mask[-1, :-2] = 1 for enable_kernels in [False, True]: failcase = f"padding_side={padding_side}, use_mask={use_mask}, batch_size={batch_size}, enable_kernels={enable_kernels}" @@ -1674,52 +1660,32 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester # Masked tokens output slightly deviates - we don't mind that. if use_mask: + _logits_sdpa = torch.zeros_like(input=logits_sdpa) + _logits_eager = torch.zeros_like(input=logits_eager) + + _logits_sdpa[:-1] = logits_sdpa[:-1] + _logits_eager[:-1] = logits_eager[:-1] + if padding_side == "left": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, 2:] + _logits_eager[-1:, 2:] = logits_eager[-1:, 2:] - sub_sdpa = logits_sdpa[-1, :-4] - sub_eager = logits_eager[-1, :-4] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) - - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, -4:] - # sub_eager = logits_eager[-1, -4:] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) elif padding_side == "right": - sub_sdpa = logits_sdpa[:-1] - sub_eager = logits_eager[:-1] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + _logits_sdpa[-1:, 2:] = logits_sdpa[-1:, :-2] + _logits_eager[-1:, 2:] = logits_eager[-1:, :-2] - sub_sdpa = logits_sdpa[-1, 3:] - sub_eager = logits_eager[-1, 3:] - if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, sub_sdpa, sub_eager, atol, rtol) - ) + logits_sdpa = _logits_sdpa + logits_eager = _logits_eager - # Testing the padding tokens is not really meaningful but anyway - # sub_sdpa = logits_sdpa[-1, :3] - # sub_eager = logits_eager[-1, :3] - # if not torch.allclose(sub_sdpa, sub_eager, atol=atol, rtol=rtol): - # fail_cases.append(get_mean_reldiff(failcase, sub_sdpa, sub_eager, 4e-2, 4e-2)) - - else: - if not torch.allclose(logits_sdpa, logits_eager, atol=atol, rtol=rtol): - fail_cases.append( - get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) - ) + results = [ + torch.allclose(_logits_sdpa, _logits_eager, atol=atol, rtol=rtol) + for (_logits_sdpa, _logits_eager) in zip(logits_sdpa, logits_eager) + ] + # If 80% batch elements have matched results, it's fine + if np.mean(results) < 0.8: + fail_cases.append( + get_mean_reldiff(failcase, logits_sdpa, logits_eager, atol, rtol) + ) self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) diff --git a/tests/models/nemotron/test_modeling_nemotron.py b/tests/models/nemotron/test_modeling_nemotron.py index 37a581a338..fd62c74d3d 100644 --- a/tests/models/nemotron/test_modeling_nemotron.py +++ b/tests/models/nemotron/test_modeling_nemotron.py @@ -19,7 +19,6 @@ import tempfile import unittest import pytest -from parameterized import parameterized from transformers import NemotronConfig, is_torch_available from transformers.testing_utils import ( @@ -99,15 +98,6 @@ class NemotronModelTest(GemmaModelTest): self.model_tester = NemotronModelTester(self) self.config_tester = ConfigTester(self, config_class=NemotronConfig, hidden_size=37) - @require_torch_sdpa - @slow - @unittest.skip( - reason="Due to custom causal mask, there is a slightly too big difference between eager and sdpa in bfloat16." - ) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - pass - @unittest.skip("Eager and SDPA do not produce the same outputs, thus this test fails") def test_model_outputs_equivalence(self, **kwargs): pass diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index 074e0083fd..ce44436a20 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -17,7 +17,6 @@ import unittest import requests -from parameterized import parameterized from transformers import ( PaliGemmaConfig, @@ -30,7 +29,6 @@ from transformers.testing_utils import ( cleanup, require_read_token, require_torch, - require_torch_sdpa, slow, torch_device, ) @@ -301,14 +299,6 @@ class PaliGemmaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTes def test_model_parallelism(self): pass - @require_torch_sdpa - @slow - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - self.skipTest( - "Due to custom causal mask, there is a slightly too big difference between eager and sdpa in bfloat16." - ) - @unittest.skip( reason="PaliGemmma's SigLip encoder uses the same initialization scheme as the Flax original implementation" ) diff --git a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py index afd45dc016..c3902c9e75 100644 --- a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py +++ b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py @@ -66,12 +66,12 @@ class Qwen2VLVisionText2TextModelTester: bos_token_id=0, eos_token_id=1, pad_token_id=2, - vision_start_token_id=151652, - image_token_id=151655, - video_token_id=151656, + vision_start_token_id=3, + image_token_id=4, + video_token_id=5, hidden_act="silu", hidden_size=32, - vocab_size=152064, + vocab_size=99, intermediate_size=37, max_position_embeddings=512, max_window_layers=3, @@ -166,6 +166,8 @@ class Qwen2VLVisionText2TextModelTester: input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device) + input_ids[:, -1] = self.pad_token_id + input_ids[input_ids == self.video_token_id] = self.pad_token_id input_ids[input_ids == self.image_token_id] = self.pad_token_id input_ids[:, self.num_image_tokens] = self.image_token_id labels = torch.zeros( diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index 4da6dc19ad..090907b164 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -57,8 +57,8 @@ class VideoLlavaVisionText2TextModelTester: image_token_index=0, video_token_index=1, projector_hidden_act="gelu", - seq_length=13, - num_frames=8, + seq_length=3, + num_frames=2, vision_feature_select_strategy="default", vision_feature_layer=-1, text_config={ @@ -88,7 +88,7 @@ class VideoLlavaVisionText2TextModelTester: vision_config={ "model_type": "clip_vision_model", "batch_size": 12, - "image_size": 30, + "image_size": 8, "patch_size": 6, "num_channels": 3, "is_training": True, @@ -123,10 +123,11 @@ class VideoLlavaVisionText2TextModelTester: self.batch_size = 5 self.num_channels = 3 self.image_size = 224 - self.encoder_seq_length = 246 - self.num_image_tokens = 25 - self.num_video_tokens = 26 * self.num_frames + + self.num_image_tokens = (vision_config["image_size"] // vision_config["patch_size"]) ** 2 + self.num_video_tokens = (self.num_image_tokens + 1) * self.num_frames self.seq_length = seq_length + self.num_image_tokens + self.num_video_tokens + self.encoder_seq_length = self.seq_length def get_config(self): return VideoLlavaConfig( diff --git a/tests/models/videomae/test_modeling_videomae.py b/tests/models/videomae/test_modeling_videomae.py index 801990331f..212eae1471 100644 --- a/tests/models/videomae/test_modeling_videomae.py +++ b/tests/models/videomae/test_modeling_videomae.py @@ -22,7 +22,7 @@ from huggingface_hub import hf_hub_download from transformers import VideoMAEConfig from transformers.models.auto import get_values -from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.testing_utils import require_torch, require_torch_sdpa, require_vision, slow, torch_device from transformers.utils import cached_property, is_torch_available, is_vision_available from ...test_configuration_common import ConfigTester @@ -213,6 +213,11 @@ class VideoMAEModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase return inputs_dict + @unittest.skip("`mse_cpu` not implemented for 'BFloat16'") + @require_torch_sdpa + def test_eager_matches_sdpa_inference_1_bfloat16(self): + pass + def test_config(self): self.config_tester.run_common_tests() diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index c7a11ff0ac..94b5e175bf 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3928,7 +3928,6 @@ class ModelTesterMixin: @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") @@ -3954,8 +3953,10 @@ class ModelTesterMixin: atols = { ("cpu", False, torch.float32): 1e-6, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-6, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-6, ("cuda", False, torch.bfloat16): 1e-2, @@ -3966,8 +3967,10 @@ class ModelTesterMixin: } rtols = { ("cpu", False, torch.float32): 1e-4, + ("cpu", False, torch.float16): 5e-3, ("cpu", False, torch.bfloat16): 1e-2, ("cpu", True, torch.float32): 1e-4, + ("cpu", True, torch.float16): 5e-3, ("cpu", True, torch.bfloat16): 1e-2, ("cuda", False, torch.float32): 1e-4, ("cuda", False, torch.bfloat16): 1e-2, @@ -3983,12 +3986,31 @@ class ModelTesterMixin: if hasattr(self.model_tester, "num_hidden_layers"): self.model_tester.num_hidden_layers = 1 if hasattr(self.model_tester, "vision_config") and "num_hidden_layers" in self.model_tester.vision_config: + self.model_tester.vision_config = copy.deepcopy(self.model_tester.vision_config) self.model_tester.vision_config["num_hidden_layers"] = 1 if hasattr(self.model_tester, "text_config") and "num_hidden_layers" in self.model_tester.text_config: + self.model_tester.text_config = copy.deepcopy(self.model_tester.text_config) self.model_tester.text_config["num_hidden_layers"] = 1 for model_class in self.all_model_classes: config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + config.rms_norm_eps = 1.0 + config.layer_norm_eps = 1.0 + config.norm_eps = 1.0 + config.norm_epsilon = 1.0 + config.layer_norm_epsilon = 1.0 + + # norm layers (layer/group norm, etc.) could cause flaky tests when the tensors have very small variance. + # (We don't need the original epsilon values to check eager/sdpa matches) + for attr in ["text_config", "vision_config", "text_encoder", "audio_encoder", "decoder"]: + if hasattr(config, attr): + getattr(config, attr).rms_norm_eps = 1.0 + getattr(config, attr).layer_norm_eps = 1.0 + getattr(config, attr).norm_eps = 1.0 + getattr(config, attr).norm_epsilon = 1.0 + getattr(config, attr).layer_norm_epsilon = 1.0 + model = model_class(config) # FIXME: we deactivate boolean mask for models using "use_mask_token" in their constructors. # These models support masking only in the case `use_mask_token=True`. Otherwise they cannot consume an input mask. @@ -4000,14 +4022,22 @@ class ModelTesterMixin: with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) - model_sdpa = model_sdpa.eval().to(torch_device) + model_sdpa = model_sdpa.eval().to(torch_device, dtype=torch_dtype) model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, attn_implementation="eager", ) - model_eager = model_eager.eval().to(torch_device) + model_eager = model_eager.eval().to(torch_device, dtype=torch_dtype) + + # Another way to make sure norm layers have desired epsilon. (Some models don't set it from its config.) + for x in model_eager.modules(): + if isinstance(x, (nn.LayerNorm, nn.GroupNorm)): + x.eps = 1.0 + for x in model_sdpa.modules(): + if isinstance(x, (nn.LayerNorm, nn.GroupNorm)): + x.eps = 1.0 # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 16 times the model, # but it would be nicer to have an efficient way to use parameterized.expand