From 1a96127e465b54048fe8ad5638bf0fc11ce94f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Ouazan?= <83456801+remi-or@users.noreply.github.com> Date: Mon, 23 Jun 2025 15:13:27 +0200 Subject: [PATCH] Break tie in Expectations and gemma3 fixes (#38943) * Added major / minor version to Expectations ordering * Added fixes to gemma3 * Style --- src/transformers/testing_utils.py | 13 +++++++++++-- tests/models/gemma3/test_modeling_gemma3.py | 12 ++++++++++++ tests/test_modeling_common.py | 4 ++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 7c3c6ccac8..cedfad084c 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -3334,9 +3334,18 @@ class Expectations(UserDict[PackedDeviceProperties, Any]): def find_expectation(self, properties: DeviceProperties = (None, None, None)) -> Any: """ - Find best matching expectation based on provided device properties. + Find best matching expectation based on provided device properties. We score each expectation, and to + distinguish between expectations with the same score, we use the major and minor version numbers, prioritizing + most recent versions. """ - (result_key, result) = max(self.unpacked(), key=lambda x: Expectations.score(properties, x[0])) + (result_key, result) = max( + self.unpacked(), + key=lambda x: ( + Expectations.score(properties, x[0]), # x[0] is a device properties tuple (device_type, major, minor) + x[0][1] if x[0][1] is not None else -1, # This key is the major version, -1 if major is None + x[0][2] if x[0][2] is not None else -1, # This key is the minor version, -1 if minor is None + ), + ) if Expectations.score(properties, result_key) == 0: raise ValueError(f"No matching expectation found for {properties}") diff --git a/tests/models/gemma3/test_modeling_gemma3.py b/tests/models/gemma3/test_modeling_gemma3.py index b0b25579b8..2c4643cf6c 100644 --- a/tests/models/gemma3/test_modeling_gemma3.py +++ b/tests/models/gemma3/test_modeling_gemma3.py @@ -409,6 +409,7 @@ class Gemma3IntegrationTest(unittest.TestCase): ("xpu", 3): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water in the background. It looks like a lovely,'], ("cuda", 7): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water in the background. It looks like a lovely,'], ("cuda", 8): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and'], + ("rocm", (9, 5)): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant coastline in the background. It looks'], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -461,6 +462,11 @@ class Gemma3IntegrationTest(unittest.TestCase): 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and', 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes:\n\n* **Image 1** shows a cow standing on a beach.', ], + ("rocm", (9, 5)): + [ + 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. There are some clouds in the blue', + 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes. \n\n* **Image 1** shows a cow standing on a beach', + ], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -558,6 +564,10 @@ class Gemma3IntegrationTest(unittest.TestCase): 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown cow standing on a sandy beach next to a turquoise ocean. There are clouds in the blue sky above.', 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nAre these images identical?\nmodel\nNo, the images are not identical. \n\nThe first image shows a cow on a beach, while the second image shows a street scene with a', ], + ("rocm", (9, 5)) : [ + 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown cow standing on a sandy beach next to a turquoise ocean. There are clouds in the blue sky above.', + 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nAre these images identical?\nmodel\nNo, the images are not identical. \n\nThe first image shows a cow on a beach, while the second image shows a street scene with a', + ], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -618,6 +628,7 @@ class Gemma3IntegrationTest(unittest.TestCase): ("xpu", 3): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a river deep,\nWith patterns hidden, secrets sleep.\nA neural net, a watchful eye,\nLearning'], ("cuda", 7): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a silent stream,\nInto the neural net, a waking dream.\nAlgorithms hum, a coded grace,\n'], ("cuda", 8): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a silent stream,\nInto the neural net, a waking dream.\nAlgorithms hum, a coded grace,\n'], + ("rocm", (9, 5)): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a river deep,\nWith patterns hidden, secrets sleep.\nA neural net, a watchful eye,\nLearning'], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -650,6 +661,7 @@ class Gemma3IntegrationTest(unittest.TestCase): ("xpu", 3): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant island in the background. It looks like a sunny day'], ("cuda", 7): [], ("cuda", 8): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant island in the background. It looks like a sunny day'], + ("rocm", (9, 5)): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown and white cow standing on a sandy beach with a turquoise ocean and a distant island in the background. It looks like a sunny'], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 874323b1e9..c482cd43d4 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3801,9 +3801,9 @@ class ModelTesterMixin: self.skipTest( "PaliGemma-like models currently (transformers==4.41.0) requires an attention_mask input" ) - if config.model_type in ["modernbert"]: + if config.model_type in ["modernbert", "gemma3"]: self.skipTest( - reason="ModernBert currently (transformers==4.52.0) automatically adds an attention_mask input" + reason=f"{config.model_type} currently (transformers==4.52.0) automatically adds an attention_mask input" ) if config.model_type in ["idefics", "idefics2", "idefics3"]: self.skipTest(reason="Idefics currently (transformers==4.39.1) requires an image_attention_mask input")