From 1168f57abffd077d7d2687087aa10ba644a76a0d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 3 Jul 2025 19:56:02 +0200 Subject: [PATCH] Update expected values (after switching to A10) - part 5 (#39205) * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix --------- Co-authored-by: ydshieh --- tests/models/emu3/test_modeling_emu3.py | 11 +++++- tests/models/encodec/test_modeling_encodec.py | 39 +++++++++++++++---- .../falcon_h1/test_modeling_falcon_h1.py | 38 +++++++++++++++++- tests/models/gemma3/test_modeling_gemma3.py | 8 ++-- tests/models/glm/test_modeling_glm.py | 6 ++- tests/models/helium/test_modeling_helium.py | 3 +- .../paligemma/test_modeling_paligemma.py | 3 +- tests/models/qwen2/test_modeling_qwen2.py | 3 ++ .../models/qwen2_vl/test_modeling_qwen2_vl.py | 4 ++ .../starcoder2/test_modeling_starcoder2.py | 19 +++++++-- 10 files changed, 111 insertions(+), 23 deletions(-) diff --git a/tests/models/emu3/test_modeling_emu3.py b/tests/models/emu3/test_modeling_emu3.py index 6d9780509b..978febaa49 100644 --- a/tests/models/emu3/test_modeling_emu3.py +++ b/tests/models/emu3/test_modeling_emu3.py @@ -404,10 +404,15 @@ class Emu3IntegrationTest(unittest.TestCase): "USER: 64*64Describe what do you see here? ASSISTANT: The image depicts a black panther in a crouched position. The panther's body is elongated and its head is lowered, suggesting a state of alertness or readiness. The animal's", "USER: 64*64What can you say about the image? ASSISTANT: The image depicts a serene natural landscape. The foreground consists of a grassy area with some patches of bare earth. The middle ground shows a gently sloping hill with a reddish-brown hue,", ], - ("cuda", 7): [ + (None, None): [ "USER: 64*64Describe what do you see here? ASSISTANT: The image depicts a black panther in a crouched position. The panther's body is elongated and curved, with its head lowered and ears pointed forward, suggesting alertness or focus.", "USER: 64*64What can you say about the image? ASSISTANT: The image depicts a serene natural landscape. The foreground consists of a grassy area with some patches of bare earth. The middle ground shows a steep, reddish-brown cliff, which could be a", ], + # We switch to A10 on 2025/06/29, and A10 gives strange values + ("cuda", 8): [ + 'USER: 64*64Describe what do you see here? ASSISTANT: 1.Filed with 1.Computing theComputing.Computing.', + 'USER: 64*64What can you say about the image? ASSISTANT: 1.Filed with theComputing theComputing.Computing.', + ], } ) # fmt: skip EXPECTED_TEXT_COMPLETION = EXPECTED_TEXT_COMPLETIONS.get_expectation() @@ -433,7 +438,9 @@ class Emu3IntegrationTest(unittest.TestCase): EXPECTED_TEXT_COMPLETIONS = Expectations( { ("xpu", 3): ['USER: 64*6464*64What do these two images have in common? ASSISTANT: The two images both depict a rhinoceros, yet they are significantly different in terms of focus and clarity. The rhinoceros in the upper image is in sharp focus, showing detailed textures'], - ("cuda", 7): ["USER: 64*6464*64What do these two images have in common? ASSISTANT: Both images feature a black animal, but they are not the same animal. The top image shows a close-up of a black cow's head, while the bottom image depicts a black cow in a natural"], + (None, None): ["USER: 64*6464*64What do these two images have in common? ASSISTANT: Both images feature a black animal, but they are not the same animal. The top image shows a close-up of a black cow's head, while the bottom image depicts a black cow in a natural"], + # We switch to A10 on 2025/06/29, and A10 gives strange values + ("cuda", 8): ['USER: 64*6464*64What do these two images have in common? ASSISTANT:Computing.Filed.Filed.11.Computing theComputing.Computing.'], } ) # fmt: skip EXPECTED_TEXT_COMPLETION = EXPECTED_TEXT_COMPLETIONS.get_expectation() diff --git a/tests/models/encodec/test_modeling_encodec.py b/tests/models/encodec/test_modeling_encodec.py index a429561b71..2f3485cbb0 100644 --- a/tests/models/encodec/test_modeling_encodec.py +++ b/tests/models/encodec/test_modeling_encodec.py @@ -24,6 +24,7 @@ from datasets import Audio, load_dataset from transformers import AutoProcessor, EncodecConfig from transformers.testing_utils import ( + Expectations, is_torch_available, require_torch, slow, @@ -459,10 +460,21 @@ class EncodecIntegrationTest(unittest.TestCase): "1.5": 0.0025, "24.0": 0.0015, } - expected_codesums = { - "1.5": [371955], - "24.0": [6659962], - } + + expectations = Expectations( + { + (None, None): { + "1.5": [371955], + "24.0": [6659962], + }, + ("cuda", 8): { + "1.5": [371955], + "24.0": [6655079], + }, + } + ) + expected_codesums = expectations.get_expectation() + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_id = "facebook/encodec_24khz" @@ -513,10 +525,21 @@ class EncodecIntegrationTest(unittest.TestCase): "3.0": 0.001, "24.0": 0.0005, } - expected_codesums = { - "3.0": [144259, 146765, 156435, 176871, 161971], - "24.0": [1568553, 1294948, 1306190, 1464747, 1663150], - } + + expectations = Expectations( + { + (None, None): { + "3.0": [144259, 146765, 156435, 176871, 161971], + "24.0": [1568553, 1294948, 1306190, 1464747, 1663150], + }, + ("cuda", 8): { + "3.0": [144259, 146765, 156205, 176871, 161971], + "24.0": [1566878, 1300459, 1310165, 1464747, 1663150], + }, + } + ) + expected_codesums = expectations.get_expectation() + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") model_id = "facebook/encodec_48khz" diff --git a/tests/models/falcon_h1/test_modeling_falcon_h1.py b/tests/models/falcon_h1/test_modeling_falcon_h1.py index 1fb85f7de8..31c9b72e28 100644 --- a/tests/models/falcon_h1/test_modeling_falcon_h1.py +++ b/tests/models/falcon_h1/test_modeling_falcon_h1.py @@ -21,6 +21,8 @@ import pytest from transformers import FalconH1Config, is_torch_available from transformers.testing_utils import ( + Expectations, + get_device_properties, require_torch, require_torch_gpu, slow, @@ -484,7 +486,7 @@ class FalconH1ModelIntegrationTest(unittest.TestCase): """ An integration test for Falcon-H1. """ - EXPECTED_TEXT = """ + EXPECTED_TEXT_DEFAULT = """ user Tell me about the french revolution. assistant @@ -503,9 +505,43 @@ class FalconH1ModelIntegrationTest(unittest.TestCase): 4. **Rise of the Jacobins and Reign of Terror (1793–1794)**: Radical leaders like Maximilien Robespierre sought to purge France of counter-revolutionaries, leading to mass executions and widespread fear. 5. **Thermidorian Reaction """ + + EXPECTED_TEXT_A10 = """ + user + Tell me about the french revolution. + assistant + The French Revolution (1789–1799) was a period of profound social upheaval and radical political change in France that fundamentally transformed the nation and had far-reaching effects on the rest of Europe and the world. Here are the key aspects of the revolution: + + ### **Causes** + 1. **Economic Crisis**: France was in severe financial trouble due to costly wars (particularly the American Revolution), extravagant spending by the monarchy, and an inefficient tax system. + 2. **Social Inequality**: The privileged classes (the nobility and clergy) enjoyed immense wealth and power, while the majority of the population (the Third Estate, comprising commoners) faced poverty and lack of representation. + 3. **Enlightenment Ideas**: Philosophers like Voltaire, Rousseau, and Montesquieu inspired ideas of liberty, equality, and popular sovereignty, which fueled revolutionary fervor. + 4. **Political Instability**: The absolute monarchy under King Louis XVI proved unable to address the nation's problems, leading to growing discontent. + + ### **Key Events** + 1. **Estates-General (1789)**: The Third Estate broke away and formed the National Assembly, forcing King Louis XVI to convene the Estates-General, an old legislative body, to address the financial crisis. + 2. **Storming of the Bastille (July 14, 1789)**: A symbol of royal tyranny, the Bastille fortress was stormed by revolutionaries, sparking widespread rebellion. + 3. **Declaration of the Rights of Man and of the Citizen (August 1789)**: This foundational document proclaimed liberty, equality, and fraternity as fundamental rights. + 4. **Abolition of Feudalism (November 1789)**: The National Assembly abolished feudal privileges, redistributing church lands to the people. + 5. **Tennis Court Oath (May 5, 1789)**: The National Assembly members, meeting on a tennis court, pledged to continue their work until a new constitution was established. + 6. + """ + + expected_texts = Expectations( + { + (None, None): EXPECTED_TEXT_DEFAULT, + ("cuda", 8): EXPECTED_TEXT_A10, + } + ) + EXPECTED_TEXT = expected_texts.get_expectation() # Remove the first char (`\n`) and the consecutive whitespaces caused by the formatting. EXPECTED_TEXT = EXPECTED_TEXT.strip().replace(" " * 12, "") + device_properties = get_device_properties() + # For A10, there is an ending " " + if device_properties[0] == "cuda" and device_properties[1] == 8: + EXPECTED_TEXT = EXPECTED_TEXT + " " + model_id = "tiiuae/Falcon-H1-1.5B-Deep-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) model = FalconH1ForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto") diff --git a/tests/models/gemma3/test_modeling_gemma3.py b/tests/models/gemma3/test_modeling_gemma3.py index 1f32df3974..8b3e12b01e 100644 --- a/tests/models/gemma3/test_modeling_gemma3.py +++ b/tests/models/gemma3/test_modeling_gemma3.py @@ -403,8 +403,7 @@ class Gemma3IntegrationTest(unittest.TestCase): EXPECTED_TEXTS = Expectations( { ("xpu", 3): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water in the background. It looks like a lovely,'], - ("cuda", 7): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water in the background. It looks like a lovely,'], - ("cuda", 8): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and'], + ("cuda", 8): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant coastline in the background. It looks'], ("rocm", (9, 5)): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant coastline in the background. It looks'], } ) # fmt: skip @@ -452,11 +451,10 @@ class Gemma3IntegrationTest(unittest.TestCase): 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and', 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes:\n\n* **Image 1** shows a cow standing on a beach.', ], - ("cuda", 7): [], ("cuda", 8): [ - 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and', - 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes:\n\n* **Image 1** shows a cow standing on a beach.', + 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant island in the background. It looks', + 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes. \n\n* **Image 1** shows a cow standing on a beach' ], ("rocm", (9, 5)): [ diff --git a/tests/models/glm/test_modeling_glm.py b/tests/models/glm/test_modeling_glm.py index 0202d9b0eb..ed1a0d4590 100644 --- a/tests/models/glm/test_modeling_glm.py +++ b/tests/models/glm/test_modeling_glm.py @@ -120,10 +120,14 @@ class GlmIntegrationTest(unittest.TestCase): def test_model_9b_eager(self): expected_texts = Expectations({ - ("cuda", None): [ + (None, None): [ "Hello I am doing a project on the history of the internetSolution:\n\nStep 1: Introduction\nThe history of the", "Hi today I am going to show you how to make a simple and easy to make a DIY paper flower.", ], + ("cuda", 8): [ + 'Hello I am doing a project on the history of the internetSolution:\n\nStep 1: Introduction\nThe history of the', + 'Hi today I am going to show you how to make a simple and easy to make a DIY paper lantern.', + ], ("rocm", (9, 5)) : [ "Hello I am doing a project on the history of the internetSolution:\n\nStep 1: Introduction\nThe history of the", "Hi today I am going to show you how to make a simple and easy to make a paper airplane. First", diff --git a/tests/models/helium/test_modeling_helium.py b/tests/models/helium/test_modeling_helium.py index d0a2f25150..b25bb7a79e 100644 --- a/tests/models/helium/test_modeling_helium.py +++ b/tests/models/helium/test_modeling_helium.py @@ -87,7 +87,8 @@ class HeliumIntegrationTest(unittest.TestCase): expected_texts = Expectations( { ("rocm", (9, 5)): ["Hello, today is a great day to start a new project. I have been working on a new project for a while now, and I"], - ("cuda", None): ["Hello, today is a great day to start a new project. I have been working on a new project for a while now and I have"], + (None, None): ["Hello, today is a great day to start a new project. I have been working on a new project for a while now and I have"], + ("cuda", 8): ['Hello, today is a great day to start a new project. I have been working on a new project for a while now, and I'], } ) # fmt: skip EXPECTED_TEXTS = expected_texts.get_expectation() diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index f75270283f..7bac1dca60 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -545,7 +545,8 @@ class PaliGemmaForConditionalGenerationIntegrationTest(unittest.TestCase): expected_decoded_texts = Expectations( { ("rocm", (9, 5)): "detect shoe\n shoe", - ("cuda", None): "detect shoe\n shoe", + (None, None): "detect shoe\n shoe", + ("cuda", 8): "detect shoe\n shoe", } ) # fmt: skip EXPECTED_DECODED_TEXT = expected_decoded_texts.get_expectation() diff --git a/tests/models/qwen2/test_modeling_qwen2.py b/tests/models/qwen2/test_modeling_qwen2.py index 59fcf240a8..fb15c21345 100644 --- a/tests/models/qwen2/test_modeling_qwen2.py +++ b/tests/models/qwen2/test_modeling_qwen2.py @@ -256,6 +256,9 @@ class Qwen2IntegrationTest(unittest.TestCase): ("cuda", None): [ "My favourite condiment is 100% natural, organic, gluten free, vegan, and free from preservatives. I" ], + ("cuda", 8): [ + "My favourite condiment is 100% natural, organic, gluten free, vegan, and vegetarian. I love to use" + ], ("rocm", (9, 5)): [ "My favourite condiment is 100% natural, organic, gluten free, vegan, and vegetarian. I love to use" ] diff --git a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py index 451f940ee0..2d5ccfa9cf 100644 --- a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py +++ b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py @@ -466,6 +466,10 @@ class Qwen2VLIntegrationTest(unittest.TestCase): 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular choices', 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets', ], + ("cuda", 8): [ + 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular choices', + 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular choices' + ], } ) # fmt: skip EXPECTED_DECODED_TEXT = EXPECTED_DECODED_TEXTS.get_expectation() diff --git a/tests/models/starcoder2/test_modeling_starcoder2.py b/tests/models/starcoder2/test_modeling_starcoder2.py index 956b210bae..274b2e4989 100644 --- a/tests/models/starcoder2/test_modeling_starcoder2.py +++ b/tests/models/starcoder2/test_modeling_starcoder2.py @@ -19,6 +19,7 @@ import pytest from transformers import Starcoder2Config, is_torch_available from transformers.testing_utils import ( + Expectations, require_bitsandbytes, require_flash_attn, require_torch, @@ -148,10 +149,20 @@ class Starcoder2IntegrationTest(unittest.TestCase): @require_bitsandbytes def test_starcoder2_batched_generation_4bit(self): - EXPECTED_TEXT = [ - 'Hello my name is Younes and I am a student at the University of Maryland. I am currently working on a project that is related to the topic of "How to make a game". I am currently working on a project', - 'def hello_world():\n\treturn "Hello World"\n\n@app.route(\'/hello/\')\ndef hello_name(name):\n\treturn "Hello " + name\n\n@app.route', - ] + expectations = Expectations( + { + (None, None): [ + 'Hello my name is Younes and I am a student at the University of Maryland. I am currently working on a project that is related to the topic of "How to make a game". I am currently working on a project', + 'def hello_world():\n\treturn "Hello World"\n\n@app.route(\'/hello/\')\ndef hello_name(name):\n\treturn "Hello " + name\n\n@app.route', + ], + ("cuda", 8): [ + "Hello my name is Younes and I am a student at the University of Maryland. I am currently working on a project that is aimed at creating a new way of learning. I am hoping to create a new way of", + 'def hello_world():\n\treturn "Hello World"\n\n@app.route(\'/hello/\')\ndef hello_name(name):\n\treturn "Hello " + name\n\n@app.route', + ], + } + ) + EXPECTED_TEXT = expectations.get_expectation() + model_id = "bigcode/starcoder2-7b" model = Starcoder2ForCausalLM.from_pretrained(model_id, load_in_4bit=True)