diff --git a/tests/models/grounding_dino/test_modeling_grounding_dino.py b/tests/models/grounding_dino/test_modeling_grounding_dino.py index d632f99e2c..953255797b 100644 --- a/tests/models/grounding_dino/test_modeling_grounding_dino.py +++ b/tests/models/grounding_dino/test_modeling_grounding_dino.py @@ -681,25 +681,48 @@ class GroundingDinoModelIntegrationTests(unittest.TestCase): expected_shape_logits = torch.Size((1, model.config.num_queries, model.config.d_model)) self.assertEqual(outputs.logits.shape, expected_shape_logits) - expected_boxes = torch.tensor( - [[0.7674, 0.4136, 0.4572], [0.2566, 0.5463, 0.4760], [0.2585, 0.5442, 0.4641]] - ).to(torch_device) - expected_logits = torch.tensor( - [[-4.8913, -0.1900, -0.2161], [-4.9653, -0.3719, -0.3950], [-5.9599, -3.3765, -3.3104]] - ).to(torch_device) + expectations = Expectations( + { + (None, None): [[0.7674, 0.4136, 0.4572], [0.2566, 0.5463, 0.4760], [0.2585, 0.5442, 0.4641]], + ("cuda", 8): [[0.7674, 0.4135, 0.4571], [0.2566, 0.5463, 0.4760], [0.2585, 0.5442, 0.4640]], + } + ) + expected_boxes = torch.tensor(expectations.get_expectation()).to(torch_device) + + expectations = Expectations( + { + (None, None): [[-4.8913, -0.1900, -0.2161], [-4.9653, -0.3719, -0.3950], [-5.9599, -3.3765, -3.3104]], + ("cuda", 8): [[-4.8927, -0.1910, -0.2169], [-4.9657, -0.3748, -0.3980], [-5.9579, -3.3812, -3.3153]], + } + ) + expected_logits = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, rtol=1e-3, atol=1e-3) expected_shape_boxes = torch.Size((1, model.config.num_queries, 4)) self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes) - torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=2e-4, atol=2e-4) # verify postprocessing results = processor.image_processor.post_process_object_detection( outputs, threshold=0.35, target_sizes=[(image.height, image.width)] )[0] - expected_scores = torch.tensor([0.4526, 0.4082]).to(torch_device) - expected_slice_boxes = torch.tensor([344.8143, 23.1796, 637.4004, 373.8295]).to(torch_device) + + expectations = Expectations( + { + (None, None): [[0.4526, 0.4082]], + ("cuda", 8): [0.4524, 0.4074], + } + ) + expected_scores = torch.tensor(expectations.get_expectation()).to(torch_device) + + expectations = Expectations( + { + (None, None): [344.8143, 23.1796, 637.4004, 373.8295], + ("cuda", 8): [344.8210, 23.1831, 637.3943, 373.8227], + } + ) + expected_slice_boxes = torch.tensor(expectations.get_expectation()).to(torch_device) self.assertEqual(len(results["scores"]), 2) torch.testing.assert_close(results["scores"], expected_scores, rtol=1e-3, atol=1e-3) diff --git a/tests/models/mask2former/test_modeling_mask2former.py b/tests/models/mask2former/test_modeling_mask2former.py index 5762a1f6ff..cf6521424b 100644 --- a/tests/models/mask2former/test_modeling_mask2former.py +++ b/tests/models/mask2former/test_modeling_mask2former.py @@ -21,6 +21,7 @@ from tests.test_modeling_common import floats_tensor from transformers import AutoModelForImageClassification, Mask2FormerConfig, is_torch_available, is_vision_available from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_4 from transformers.testing_utils import ( + Expectations, require_timm, require_torch, require_torch_accelerator, @@ -403,7 +404,7 @@ class Mask2FormerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestC ) -TOLERANCE = 1e-4 +TOLERANCE = 2e-4 # We will verify our results on an image of cute cats @@ -438,31 +439,52 @@ class Mask2FormerModelIntegrationTest(unittest.TestCase): outputs = model(**inputs) expected_slice_hidden_state = torch.tensor( - [[-0.2790, -1.0717, -1.1668], [-0.5128, -0.3128, -0.4987], [-0.5832, 0.1971, -0.0197]] + [ + [-0.2790, -1.0717, -1.1668], + [-0.5128, -0.3128, -0.4987], + [-0.5832, 0.1971, -0.0197], + ] ).to(torch_device) - self.assertTrue( - torch.allclose( - outputs.encoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE - ) + torch.testing.assert_close( + outputs.encoder_last_hidden_state[0, 0, :3, :3], + expected_slice_hidden_state, + atol=TOLERANCE, + rtol=TOLERANCE, ) - expected_slice_hidden_state = torch.tensor( - [[0.8973, 1.1847, 1.1776], [1.1934, 1.5040, 1.5128], [1.1153, 1.4486, 1.4951]] - ).to(torch_device) - self.assertTrue( - torch.allclose( - outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE - ) + expectations = Expectations( + { + (None, None): [ + [0.8973, 1.1847, 1.1776], + [1.1934, 1.5040, 1.5128], + [1.1153, 1.4486, 1.4951], + ], + ("cuda", 8): [ + [0.8974, 1.1848, 1.1777], + [1.1933, 1.5041, 1.5128], + [1.1154, 1.4487, 1.4950], + ], + } ) + expected_slice_hidden_state = torch.tensor(expectations.get_expectation()).to(torch_device) + torch.testing.assert_close(outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE,rtol=TOLERANCE) # fmt: skip - expected_slice_hidden_state = torch.tensor( - [[2.1152, 1.7000, -0.8603], [1.5808, 1.8004, -0.9353], [1.6043, 1.7495, -0.5999]] - ).to(torch_device) - self.assertTrue( - torch.allclose( - outputs.transformer_decoder_last_hidden_state[0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE - ) + expectations = Expectations( + { + (None, None): [ + [2.1152, 1.7000, -0.8603], + [1.5808, 1.8004, -0.9353], + [1.6043, 1.7495, -0.5999], + ], + ("cuda", 8): [ + [2.1153, 1.7004, -0.8604], + [1.5807, 1.8007, -0.9354], + [1.6040, 1.7498, -0.6001], + ], + } ) + expected_slice_hidden_state = torch.tensor(expectations.get_expectation()).to(torch_device) + torch.testing.assert_close(outputs.transformer_decoder_last_hidden_state[0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE, rtol=TOLERANCE) # fmt: skip def test_inference_universal_segmentation_head(self): model = Mask2FormerForUniversalSegmentation.from_pretrained(self.model_checkpoints).to(torch_device).eval() @@ -482,23 +504,40 @@ class Mask2FormerModelIntegrationTest(unittest.TestCase): self.assertEqual( masks_queries_logits.shape, (1, model.config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4) ) - expected_slice = [ - [-8.7839, -9.0056, -8.8121], - [-7.4104, -7.0313, -6.5401], - [-6.6105, -6.3427, -6.4675], - ] - expected_slice = torch.tensor(expected_slice).to(torch_device) + expectations = Expectations( + { + (None, None): [ + [-8.7839, -9.0056, -8.8121], + [-7.4104, -7.0313, -6.5401], + [-6.6105, -6.3427, -6.4675], + ], + ("cuda", 8): [ + [-8.7809, -9.0041, -8.8087], + [-7.4075, -7.0307, -6.5385], + [-6.6088, -6.3417, -6.4627], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close(masks_queries_logits[0, 0, :3, :3], expected_slice, rtol=TOLERANCE, atol=TOLERANCE) # class_queries_logits class_queries_logits = outputs.class_queries_logits self.assertEqual(class_queries_logits.shape, (1, model.config.num_queries, model.config.num_labels + 1)) - expected_slice = torch.tensor( - [ - [1.8324, -8.0835, -4.1922], - [0.8450, -9.0050, -3.6053], - [0.3045, -7.7293, -3.0275], - ] - ).to(torch_device) + expectations = Expectations( + { + (None, None): [ + [1.8324, -8.0835, -4.1922], + [0.8450, -9.0050, -3.6053], + [0.3045, -7.7293, -3.0275], + ], + ("cuda", 8): [ + [1.8326, -8.0834, -4.1916], + [0.8446, -9.0048, -3.6048], + [0.3042, -7.7296, -3.0277], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close( outputs.class_queries_logits[0, :3, :3], expected_slice, rtol=TOLERANCE, atol=TOLERANCE ) diff --git a/tests/models/maskformer/test_modeling_maskformer.py b/tests/models/maskformer/test_modeling_maskformer.py index 2f30d4dc3c..8644439f4a 100644 --- a/tests/models/maskformer/test_modeling_maskformer.py +++ b/tests/models/maskformer/test_modeling_maskformer.py @@ -21,6 +21,7 @@ import numpy as np from tests.test_modeling_common import floats_tensor from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available from transformers.testing_utils import ( + Expectations, require_timm, require_torch, require_torch_accelerator, @@ -478,7 +479,7 @@ class MaskFormerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa self.assertEqual(model.model.pixel_level_module.encoder.out_indices, [1, 2, 3]) -TOLERANCE = 1e-4 +TOLERANCE = 2e-4 # We will verify our results on an image of cute cats @@ -513,31 +514,43 @@ class MaskFormerModelIntegrationTest(unittest.TestCase): outputs = model(**inputs) expected_slice_hidden_state = torch.tensor( - [[-0.0482, 0.9228, 0.4951], [-0.2547, 0.8017, 0.8527], [-0.0069, 0.3385, -0.0089]] + [ + [-0.0482, 0.9228, 0.4951], + [-0.2547, 0.8017, 0.8527], + [-0.0069, 0.3385, -0.0089], + ] ).to(torch_device) - self.assertTrue( - torch.allclose( - outputs.encoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE - ) - ) + torch.allclose(outputs.encoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE, rtol=TOLERANCE) # fmt: skip - expected_slice_hidden_state = torch.tensor( - [[-0.8422, -0.8434, -0.9718], [-1.0144, -0.5565, -0.4195], [-1.0038, -0.4484, -0.1961]] - ).to(torch_device) - self.assertTrue( - torch.allclose( - outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE - ) + expectations = Expectations( + { + (None, None): [[-0.8422, -0.8434, -0.9718], [-1.0144, -0.5565, -0.4195], [-1.0038, -0.4484, -0.1961]], + ("cuda", 8): [ + [-0.8422, -0.8435, -0.9717], + [-1.0145, -0.5564, -0.4195], + [-1.0040, -0.4486, -0.1962], + ], + } ) + expected_slice_hidden_state = torch.tensor(expectations.get_expectation()).to(torch_device) + torch.allclose(outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE,rtol=TOLERANCE) # fmt: skip - expected_slice_hidden_state = torch.tensor( - [[0.2852, -0.0159, 0.9735], [0.6254, 0.1858, 0.8529], [-0.0680, -0.4116, 1.8413]] - ).to(torch_device) - self.assertTrue( - torch.allclose( - outputs.transformer_decoder_last_hidden_state[0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE - ) + expectations = Expectations( + { + (None, None): [ + [0.2852, -0.0159, 0.9735], + [0.6254, 0.1858, 0.8529], + [-0.0680, -0.4116, 1.8413], + ], + ("cuda", 8): [ + [0.2853, -0.0162, 0.9736], + [0.6256, 0.1856, 0.8530], + [-0.0679, -0.4118, 1.8416], + ], + } ) + expected_slice_hidden_state = torch.tensor(expectations.get_expectation()).to(torch_device) + torch.allclose(outputs.transformer_decoder_last_hidden_state[0, :3, :3], expected_slice_hidden_state, atol=TOLERANCE, rtol=TOLERANCE) # fmt: skip def test_inference_instance_segmentation_head(self): model = ( @@ -562,25 +575,42 @@ class MaskFormerModelIntegrationTest(unittest.TestCase): masks_queries_logits.shape, (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4), ) - expected_slice = [ - [-1.3737124, -1.7724937, -1.9364233], - [-1.5977281, -1.9867939, -2.1523695], - [-1.5795398, -1.9269832, -2.093942], - ] - expected_slice = torch.tensor(expected_slice).to(torch_device) + expectations = Expectations( + { + (None, None): [ + [-1.3737124, -1.7724937, -1.9364233], + [-1.5977281, -1.9867939, -2.1523695], + [-1.5795398, -1.9269832, -2.093942], + ], + ("cuda", 8): [ + [-1.3737, -1.7727, -1.9367], + [-1.5979, -1.9871, -2.1527], + [-1.5797, -1.9271, -2.0941], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close(masks_queries_logits[0, 0, :3, :3], expected_slice, rtol=TOLERANCE, atol=TOLERANCE) # class_queries_logits class_queries_logits = outputs.class_queries_logits self.assertEqual( class_queries_logits.shape, (1, model.config.decoder_config.num_queries, model.config.num_labels + 1) ) - expected_slice = torch.tensor( - [ - [1.6512e00, -5.2572e00, -3.3519e00], - [3.6169e-02, -5.9025e00, -2.9313e00], - [1.0766e-04, -7.7630e00, -5.1263e00], - ] - ).to(torch_device) + expectations = Expectations( + { + (None, None): [ + [1.6512e00, -5.2572e00, -3.3519e00], + [3.6169e-02, -5.9025e00, -2.9313e00], + [1.0766e-04, -7.7630e00, -5.1263e00], + ], + ("cuda", 8): [ + [1.6507e00, -5.2568e00, -3.3520e00], + [3.5767e-02, -5.9023e00, -2.9313e00], + [-6.2712e-04, -7.7627e00, -5.1268e00], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close( outputs.class_queries_logits[0, :3, :3], expected_slice, rtol=TOLERANCE, atol=TOLERANCE ) @@ -608,17 +638,34 @@ class MaskFormerModelIntegrationTest(unittest.TestCase): masks_queries_logits.shape, (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4), ) - expected_slice = [[-0.9046, -2.6366, -4.6062], [-3.4179, -5.7890, -8.8057], [-4.9179, -7.6560, -10.7711]] - expected_slice = torch.tensor(expected_slice).to(torch_device) + expectations = Expectations( + { + (None, None): [[-0.9046, -2.6366, -4.6062], [-3.4179, -5.7890, -8.8057], [-4.9179, -7.6560, -10.7711]], + ("cuda", 8): [[-0.9000, -2.6283, -4.5964], [-3.4123, -5.7789, -8.7919], [-4.9132, -7.6444, -10.7557]], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close(masks_queries_logits[0, 0, :3, :3], expected_slice, rtol=TOLERANCE, atol=TOLERANCE) # class_queries_logits class_queries_logits = outputs.class_queries_logits self.assertEqual( class_queries_logits.shape, (1, model.config.decoder_config.num_queries, model.config.num_labels + 1) ) - expected_slice = torch.tensor( - [[4.7188, -3.2585, -2.8857], [6.6871, -2.9181, -1.2487], [7.2449, -2.2764, -2.1874]] - ).to(torch_device) + expectations = Expectations( + { + (None, None): [ + [4.7188, -3.2585, -2.8857], + [6.6871, -2.9181, -1.2487], + [7.2449, -2.2764, -2.1874], + ], + ("cuda", 8): [ + [4.7177, -3.2586, -2.8853], + [6.6845, -2.9186, -1.2491], + [7.2443, -2.2760, -2.1858], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) torch.testing.assert_close( outputs.class_queries_logits[0, :3, :3], expected_slice, rtol=TOLERANCE, atol=TOLERANCE ) diff --git a/tests/models/mobilenet_v1/test_modeling_mobilenet_v1.py b/tests/models/mobilenet_v1/test_modeling_mobilenet_v1.py index 7c05d4b41c..688542c727 100644 --- a/tests/models/mobilenet_v1/test_modeling_mobilenet_v1.py +++ b/tests/models/mobilenet_v1/test_modeling_mobilenet_v1.py @@ -16,7 +16,7 @@ import unittest from transformers import MobileNetV1Config -from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.testing_utils import Expectations, require_torch, require_vision, slow, torch_device from transformers.utils import cached_property, is_torch_available, is_vision_available from ...test_configuration_common import ConfigTester @@ -246,6 +246,12 @@ class MobileNetV1ModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 1001)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice = torch.tensor([-4.1739, -1.1233, 3.1205]).to(torch_device) + expectations = Expectations( + { + (None, None): [-4.1739, -1.1233, 3.1205], + ("cuda", 8): [-4.1725, -1.1238, 3.1191], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=2e-4, atol=2e-4) diff --git a/tests/models/mobilenet_v2/test_modeling_mobilenet_v2.py b/tests/models/mobilenet_v2/test_modeling_mobilenet_v2.py index 13c7698af5..5f3807cda8 100644 --- a/tests/models/mobilenet_v2/test_modeling_mobilenet_v2.py +++ b/tests/models/mobilenet_v2/test_modeling_mobilenet_v2.py @@ -16,7 +16,7 @@ import unittest from transformers import MobileNetV2Config -from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.testing_utils import Expectations, require_torch, require_vision, slow, torch_device from transformers.utils import cached_property, is_torch_available, is_vision_available from ...test_configuration_common import ConfigTester @@ -301,9 +301,15 @@ class MobileNetV2ModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 1001)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice = torch.tensor([0.2445, -1.1993, 0.1905]).to(torch_device) + expectations = Expectations( + { + (None, None): [0.2445, -1.1993, 0.1905], + ("cuda", 8): [0.2445, -1.1970, 0.1868], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=2e-4, atol=2e-4) @slow def test_inference_semantic_segmentation(self): @@ -324,13 +330,20 @@ class MobileNetV2ModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 21, 65, 65)) self.assertEqual(logits.shape, expected_shape) - expected_slice = torch.tensor( - [ - [[17.5790, 17.7581, 18.3355], [18.3257, 18.4230, 18.8973], [18.6169, 18.8650, 19.2187]], - [[-2.1595, -2.0977, -2.3741], [-2.4226, -2.3028, -2.6835], [-2.7819, -2.5991, -2.7706]], - [[4.2058, 4.8317, 4.7638], [4.4136, 5.0361, 4.9383], [4.5028, 4.9644, 4.8734]], - ], - device=torch_device, + expectations = Expectations( + { + (None, None): [ + [[17.5790, 17.7581, 18.3355], [18.3257, 18.4230, 18.8973], [18.6169, 18.8650, 19.2187]], + [[-2.1595, -2.0977, -2.3741], [-2.4226, -2.3028, -2.6835], [-2.7819, -2.5991, -2.7706]], + [[4.2058, 4.8317, 4.7638], [4.4136, 5.0361, 4.9383], [4.5028, 4.9644, 4.8734]], + ], + ("cuda", 8): [ + [[17.5809, 17.7571, 18.3341], [18.3240, 18.4216, 18.8974], [18.6174, 18.8662, 19.2177]], + [[-2.1562, -2.0942, -2.3703], [-2.4199, -2.2999, -2.6818], [-2.7800, -2.5944, -2.7678]], + [[4.2092, 4.8356, 4.7694], [4.4181, 5.0401, 4.9409], [4.5089, 4.9700, 4.8802]], + ], + } ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(logits[0, :3, :3, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(logits[0, :3, :3, :3], expected_slice, rtol=2e-4, atol=2e-4) diff --git a/tests/models/mobilevit/test_modeling_mobilevit.py b/tests/models/mobilevit/test_modeling_mobilevit.py index f6cc09eddd..43fb0d638e 100644 --- a/tests/models/mobilevit/test_modeling_mobilevit.py +++ b/tests/models/mobilevit/test_modeling_mobilevit.py @@ -16,7 +16,7 @@ import unittest from transformers import MobileViTConfig -from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.testing_utils import Expectations, require_torch, require_vision, slow, torch_device from transformers.utils import cached_property, is_torch_available, is_vision_available from ...test_configuration_common import ConfigTester @@ -304,9 +304,15 @@ class MobileViTModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 1000)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice = torch.tensor([-1.9364, -1.2327, -0.4653]).to(torch_device) + expectations = Expectations( + { + (None, None): [-1.9364, -1.2327, -0.4653], + ("cuda", 8): [-1.9401, -1.2384, -0.4702], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=2e-4, atol=2e-4) @slow def test_inference_semantic_segmentation(self): @@ -327,16 +333,23 @@ class MobileViTModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 21, 32, 32)) self.assertEqual(logits.shape, expected_shape) - expected_slice = torch.tensor( - [ - [[6.9713, 6.9786, 7.2422], [7.2893, 7.2825, 7.4446], [7.6580, 7.8797, 7.9420]], - [[-10.6869, -10.3250, -10.3471], [-10.4228, -9.9868, -9.7132], [-11.0405, -11.0221, -10.7318]], - [[-3.3089, -2.8539, -2.6740], [-3.2706, -2.5621, -2.5108], [-3.2534, -2.6615, -2.6651]], - ], - device=torch_device, + expectations = Expectations( + { + (None, None): [ + [[6.9713, 6.9786, 7.2422], [7.2893, 7.2825, 7.4446], [7.6580, 7.8797, 7.9420]], + [[-10.6869, -10.3250, -10.3471], [-10.4228, -9.9868, -9.7132], [-11.0405, -11.0221, -10.7318]], + [[-3.3089, -2.8539, -2.6740], [-3.2706, -2.5621, -2.5108], [-3.2534, -2.6615, -2.6651]], + ], + ("cuda", 8): [ + [[6.9661, 6.9753, 7.2386], [7.2864, 7.2785, 7.4429], [7.6577, 7.8770, 7.9387]], + [[-10.7046, -10.3411, -10.3641], [-10.4402, -10.0004, -9.7269], [-11.0579, -11.0358, -10.7459]], + [[-3.3022, -2.8465, -2.6661], [-3.2654, -2.5542, -2.5055], [-3.2477, -2.6544, -2.6562]], + ], + } ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(logits[0, :3, :3, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(logits[0, :3, :3, :3], expected_slice, rtol=2e-4, atol=2e-4) @slow def test_post_processing_semantic_segmentation(self): diff --git a/tests/models/mobilevitv2/test_modeling_mobilevitv2.py b/tests/models/mobilevitv2/test_modeling_mobilevitv2.py index 7a0433f123..daca2394be 100644 --- a/tests/models/mobilevitv2/test_modeling_mobilevitv2.py +++ b/tests/models/mobilevitv2/test_modeling_mobilevitv2.py @@ -16,7 +16,14 @@ import unittest from transformers import MobileViTV2Config -from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device +from transformers.testing_utils import ( + Expectations, + require_torch, + require_torch_multi_gpu, + require_vision, + slow, + torch_device, +) from transformers.utils import cached_property, is_torch_available, is_vision_available from ...test_configuration_common import ConfigTester @@ -317,9 +324,15 @@ class MobileViTV2ModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 1000)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice = torch.tensor([-1.6336e00, -7.3204e-02, -5.1883e-01]).to(torch_device) + expectations = Expectations( + { + (None, None): [-1.6336e00, -7.3204e-02, -5.1883e-01], + ("cuda", 8): [-1.6341, -0.0665, -0.5158], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(outputs.logits[0, :3], expected_slice, rtol=2e-4, atol=2e-4) @slow def test_inference_semantic_segmentation(self): @@ -340,16 +353,23 @@ class MobileViTV2ModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, 21, 32, 32)) self.assertEqual(logits.shape, expected_shape) - expected_slice = torch.tensor( - [ - [[7.0863, 7.1525, 6.8201], [6.6931, 6.8770, 6.8933], [6.2978, 7.0366, 6.9636]], - [[-3.7134, -3.6712, -3.6675], [-3.5825, -3.3549, -3.4777], [-3.3435, -3.3979, -3.2857]], - [[-2.9329, -2.8003, -2.7369], [-3.0564, -2.4780, -2.0207], [-2.6889, -1.9298, -1.7640]], - ], - device=torch_device, + expectations = Expectations( + { + (None, None): [ + [[7.0863, 7.1525, 6.8201], [6.6931, 6.8770, 6.8933], [6.2978, 7.0366, 6.9636]], + [[-3.7134, -3.6712, -3.6675], [-3.5825, -3.3549, -3.4777], [-3.3435, -3.3979, -3.2857]], + [[-2.9329, -2.8003, -2.7369], [-3.0564, -2.4780, -2.0207], [-2.6889, -1.9298, -1.7640]], + ], + ("cuda", 8): [ + [[7.0866, 7.1509, 6.8188], [6.6935, 6.8757, 6.8927], [6.2988, 7.0365, 6.9631]], + [[-3.7113, -3.6686, -3.6643], [-3.5801, -3.3516, -3.4739], [-3.3432, -3.3966, -3.2832]], + [[-2.9359, -2.8037, -2.7387], [-3.0595, -2.4798, -2.0222], [-2.6901, -1.9306, -1.7659]], + ], + } ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(logits[0, :3, :3, :3], expected_slice, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(logits[0, :3, :3, :3], expected_slice, rtol=2e-4, atol=2e-4) @slow def test_post_processing_semantic_segmentation(self): diff --git a/tests/models/rt_detr/test_modeling_rt_detr.py b/tests/models/rt_detr/test_modeling_rt_detr.py index fa2938160d..fad9093426 100644 --- a/tests/models/rt_detr/test_modeling_rt_detr.py +++ b/tests/models/rt_detr/test_modeling_rt_detr.py @@ -29,6 +29,7 @@ from transformers import ( is_vision_available, ) from transformers.testing_utils import ( + Expectations, require_torch, require_torch_accelerator, require_vision, @@ -732,45 +733,69 @@ class RTDetrModelIntegrationTest(unittest.TestCase): expected_shape_logits = torch.Size((1, 300, model.config.num_labels)) self.assertEqual(outputs.logits.shape, expected_shape_logits) - expected_logits = torch.tensor( - [ - [-4.64763879776001, -5.001153945922852, -4.978509902954102], - [-4.159348487854004, -4.703853607177734, -5.946484565734863], - [-4.437461853027344, -4.65836238861084, -6.235235691070557], - ] - ).to(torch_device) - expected_boxes = torch.tensor( - [ - [0.1688060760498047, 0.19992263615131378, 0.21225441992282867], - [0.768376350402832, 0.41226309537887573, 0.4636859893798828], - [0.25953856110572815, 0.5483334064483643, 0.4777486026287079], - ] - ).to(torch_device) + expectations = Expectations( + { + (None, None): [ + [-4.64763879776001, -5.001153945922852, -4.978509902954102], + [-4.159348487854004, -4.703853607177734, -5.946484565734863], + [-4.437461853027344, -4.65836238861084, -6.235235691070557], + ], + ("cuda", 8): [[-4.6471, -5.0008, -4.9786], [-4.1599, -4.7041, -5.9458], [-4.4374, -4.6582, -6.2340]], + } + ) + expected_logits = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, rtol=1e-4, atol=1e-4) + expectations = Expectations( + { + (None, None): [ + [0.1688060760498047, 0.19992263615131378, 0.21225441992282867], + [0.768376350402832, 0.41226309537887573, 0.4636859893798828], + [0.25953856110572815, 0.5483334064483643, 0.4777486026287079], + ], + ("cuda", 8): [[0.1688, 0.1999, 0.2123], [0.7684, 0.4123, 0.4637], [0.2596, 0.5483, 0.4777]], + } + ) + expected_boxes = torch.tensor(expectations.get_expectation()).to(torch_device) + + torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, rtol=2e-4, atol=2e-4) expected_shape_boxes = torch.Size((1, 300, 4)) self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes) - torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, rtol=2e-4, atol=2e-4) # verify postprocessing results = image_processor.post_process_object_detection( outputs, threshold=0.0, target_sizes=[image.size[::-1]] )[0] - expected_scores = torch.tensor( - [0.9703017473220825, 0.9599503874778748, 0.9575679302215576, 0.9506784677505493], device=torch_device - ) - expected_labels = [57, 15, 15, 65] - expected_slice_boxes = torch.tensor( - [ - [0.13774872, 0.37821293, 640.13074, 476.21088], - [343.38132, 24.276838, 640.1404, 371.49573], - [13.225126, 54.179348, 318.98422, 472.2207], - [40.114475, 73.44104, 175.9573, 118.48469], - ], - device=torch_device, - ) - torch.testing.assert_close(results["scores"][:4], expected_scores, rtol=1e-4, atol=1e-4) + expectations = Expectations( + { + (None, None): [0.9703017473220825, 0.9599503874778748, 0.9575679302215576, 0.9506784677505493], + ("cuda", 8): [0.9704, 0.9599, 0.9576, 0.9507], + } + ) + expected_scores = torch.tensor(expectations.get_expectation()).to(torch_device) + + expected_labels = [57, 15, 15, 65] + + expectations = Expectations( + { + (None, None): [ + [0.13774872, 0.37821293, 640.13074, 476.21088], + [343.38132, 24.276838, 640.1404, 371.49573], + [13.225126, 54.179348, 318.98422, 472.2207], + [40.114475, 73.44104, 175.9573, 118.48469], + ], + ("cuda", 8): [ + [1.4183e-01, 3.8063e-01, 6.4013e02, 4.7621e02], + [3.4338e02, 2.4275e01, 6.4014e02, 3.7150e02], + [1.3236e01, 5.4179e01, 3.1899e02, 4.7222e02], + [4.0114e01, 7.3441e01, 1.7596e02, 1.1848e02], + ], + } + ) + expected_slice_boxes = torch.tensor(expectations.get_expectation()).to(torch_device) + + torch.testing.assert_close(results["scores"][:4], expected_scores, rtol=2e-4, atol=2e-4) self.assertSequenceEqual(results["labels"][:4].tolist(), expected_labels) - torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=2e-4, atol=2e-4) diff --git a/tests/models/rt_detr_v2/test_modeling_rt_detr_v2.py b/tests/models/rt_detr_v2/test_modeling_rt_detr_v2.py index a78f11ea46..79202d3cf7 100644 --- a/tests/models/rt_detr_v2/test_modeling_rt_detr_v2.py +++ b/tests/models/rt_detr_v2/test_modeling_rt_detr_v2.py @@ -28,6 +28,7 @@ from transformers import ( is_vision_available, ) from transformers.testing_utils import ( + Expectations, require_torch, require_torch_accelerator, require_vision, @@ -736,42 +737,60 @@ class RTDetrV2ModelIntegrationTest(unittest.TestCase): expected_shape_logits = torch.Size((1, 300, model.config.num_labels)) self.assertEqual(outputs.logits.shape, expected_shape_logits) - expected_logits = torch.tensor( - [ - [-3.7047, -5.1914, -6.1787], - [-4.0108, -9.3449, -5.2047], - [-4.1287, -4.7461, -5.8633], - ] - ).to(torch_device) - expected_boxes = torch.tensor( - [ - [0.2582, 0.5497, 0.4764], - [0.1684, 0.1985, 0.2120], - [0.7665, 0.4146, 0.4669], - ] - ).to(torch_device) + expectations = Expectations( + { + (None, None): [[-3.7047, -5.1914, -6.1787], [-4.0108, -9.3449, -5.2047], [-4.1287, -4.7461, -5.8633]], + ("cuda", 8): [[-3.7039, -5.1923, -6.1787], [-4.0106, -9.3452, -5.2045], [-4.1285, -4.7468, -5.8641]], + } + ) + expected_logits = torch.tensor(expectations.get_expectation()).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, atol=1e-4, rtol=1e-4) + expectations = Expectations( + { + (None, None): [[0.2582, 0.5497, 0.4764], [0.1684, 0.1985, 0.2120], [0.7665, 0.4146, 0.4669]], + } + ) + expected_boxes = torch.tensor(expectations.get_expectation()).to(torch_device) + + torch.testing.assert_close(outputs.logits[0, :3, :3], expected_logits, atol=2e-4, rtol=2e-4) expected_shape_boxes = torch.Size((1, 300, 4)) self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes) - torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_boxes, atol=2e-4, rtol=2e-4) # verify postprocessing results = image_processor.post_process_object_detection( outputs, threshold=0.0, target_sizes=[image.size[::-1]] )[0] - expected_scores = torch.tensor([0.9652, 0.9599, 0.9462, 0.8613], device=torch_device) - expected_labels = [15, 15, 65, 57] - expected_slice_boxes = torch.tensor( - [ - [3.4114e02, 2.5111e01, 6.3998e02, 3.7289e02], - [1.2780e01, 5.6346e01, 3.1767e02, 4.7134e02], - [3.9959e01, 7.3117e01, 1.7565e02, 1.1744e02], - [-1.0521e-01, 2.9717e00, 6.3989e02, 4.7362e02], - ], - device=torch_device, + + expectations = Expectations( + { + (None, None): [0.9652, 0.9599, 0.9462, 0.8613], + ("cuda", 8): [0.9652, 0.9599, 0.9461, 0.8613], + } ) - self.assertTrue(torch.allclose(results["scores"][:4], expected_scores, atol=1e-3, rtol=1e-4)) + expected_scores = torch.tensor(expectations.get_expectation()).to(torch_device) + + expected_labels = [15, 15, 65, 57] + + expectations = Expectations( + { + (None, None): [ + [3.4114e02, 2.5111e01, 6.3998e02, 3.7289e02], + [1.2780e01, 5.6346e01, 3.1767e02, 4.7134e02], + [3.9959e01, 7.3117e01, 1.7565e02, 1.1744e02], + [-1.0521e-01, 2.9717e00, 6.3989e02, 4.7362e02], + ], + ("cuda", 8): [ + [3.4115e02, 2.5109e01, 6.3997e02, 3.7290e02], + [1.2785e01, 5.6350e01, 3.1767e02, 4.7134e02], + [3.9959e01, 7.3117e01, 1.7565e02, 1.1744e02], + [-1.0471e-01, 2.9680e00, 6.3989e02, 4.7362e02], + ], + } + ) + expected_slice_boxes = torch.tensor(expectations.get_expectation()).to(torch_device) + + torch.testing.assert_close(results["scores"][:4], expected_scores, atol=1e-3, rtol=2e-4) self.assertSequenceEqual(results["labels"][:4].tolist(), expected_labels) - torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, atol=1e-3, rtol=1e-4) + torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, atol=1e-3, rtol=2e-4) diff --git a/tests/models/segformer/test_modeling_segformer.py b/tests/models/segformer/test_modeling_segformer.py index cd75545c62..fcd6594217 100644 --- a/tests/models/segformer/test_modeling_segformer.py +++ b/tests/models/segformer/test_modeling_segformer.py @@ -16,7 +16,7 @@ import unittest from transformers import SegformerConfig, is_torch_available, is_vision_available -from transformers.testing_utils import require_torch, slow, torch_device +from transformers.testing_utils import Expectations, require_torch, slow, torch_device from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor @@ -200,6 +200,9 @@ class SegformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCas config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_for_image_segmentation(*config_and_inputs) + def test_batching_equivalence(self, atol=2e-4, rtol=2e-4): + super().test_batching_equivalence(atol=atol, rtol=rtol) + @unittest.skip(reason="SegFormer does not use inputs_embeds") def test_inputs_embeds(self): pass @@ -367,14 +370,22 @@ class SegformerModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, model.config.num_labels, 128, 128)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice = torch.tensor( - [ - [[-4.6310, -5.5232, -6.2356], [-5.1921, -6.1444, -6.5996], [-5.4424, -6.2790, -6.7574]], - [[-12.1391, -13.3122, -13.9554], [-12.8732, -13.9352, -14.3563], [-12.9438, -13.8226, -14.2513]], - [[-12.5134, -13.4686, -14.4915], [-12.8669, -14.4343, -14.7758], [-13.2523, -14.5819, -15.0694]], - ] - ).to(torch_device) - torch.testing.assert_close(outputs.logits[0, :3, :3, :3], expected_slice, rtol=1e-4, atol=1e-4) + expectations = Expectations( + { + (None, None): [ + [[-4.6310, -5.5232, -6.2356], [-5.1921, -6.1444, -6.5996], [-5.4424, -6.2790, -6.7574]], + [[-12.1391, -13.3122, -13.9554], [-12.8732, -13.9352, -14.3563], [-12.9438, -13.8226, -14.2513]], + [[-12.5134, -13.4686, -14.4915], [-12.8669, -14.4343, -14.7758], [-13.2523, -14.5819, -15.0694]], + ], + ("cuda", 8): [ + [[-4.6310, -5.5232, -6.2361], [-5.1918, -6.1445, -6.5996], [-5.4427, -6.2792, -6.7580]], + [[-12.1397, -13.3124, -13.9551], [-12.8736, -13.9347, -14.3569], [-12.9440, -13.8222, -14.2514]], + [[-12.5135, -13.4682, -14.4913], [-12.8670, -14.4339, -14.7766], [-13.2519, -14.5800, -15.0685]], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) + torch.testing.assert_close(outputs.logits[0, :3, :3, :3], expected_slice, rtol=2e-4, atol=2e-4) @slow def test_inference_image_segmentation_city(self): @@ -396,13 +407,24 @@ class SegformerModelIntegrationTest(unittest.TestCase): expected_shape = torch.Size((1, model.config.num_labels, 128, 128)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice = torch.tensor( - [ - [[-13.5748, -13.9111, -12.6500], [-14.3500, -15.3683, -14.2328], [-14.7532, -16.0424, -15.6087]], - [[-17.1651, -15.8725, -12.9653], [-17.2580, -17.3718, -14.8223], [-16.6058, -16.8783, -16.7452]], - [[-3.6456, -3.0209, -1.4203], [-3.0797, -3.1959, -2.0000], [-1.8757, -1.9217, -1.6997]], - ] - ).to(torch_device) + expected_slice = torch.tensor([]).to(torch_device) + + expectations = Expectations( + { + (None, None): [ + [[-13.5748, -13.9111, -12.6500], [-14.3500, -15.3683, -14.2328], [-14.7532, -16.0424, -15.6087]], + [[-17.1651, -15.8725, -12.9653], [-17.2580, -17.3718, -14.8223], [-16.6058, -16.8783, -16.7452]], + [[-3.6456, -3.0209, -1.4203], [-3.0797, -3.1959, -2.0000], [-1.8757, -1.9217, -1.6997]], + ], + ("cuda", 8): [ + [[-13.5728, -13.9089, -12.6492], [-14.3478, -15.3656, -14.2309], [-14.7512, -16.0394, -15.6065]], + [[-17.1642, -15.8704, -12.9641], [-17.2572, -17.3701, -14.8214], [-16.6043, -16.8761, -16.7425]], + [[-3.6444, -3.0189, -1.4195], [-3.0787, -3.1953, -1.9993], [-1.8755, -1.9219, -1.7002]], + ], + } + ) + expected_slice = torch.tensor(expectations.get_expectation()).to(torch_device) + torch.testing.assert_close(outputs.logits[0, :3, :3, :3], expected_slice, rtol=1e-1, atol=1e-1) @slow