SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Bug fix gguf qwen2moe (#33940)

Browse Source 
* fix qwen2moe tensors mapping, add unit tests

* add expert tensor split logic, test refactoring

* small params refactoring

* add comment to tensor reshaping
This commit is contained in:
Vladislav Bronzov
2024-10-05 16:19:01 +02:00
committed by GitHub
parent 56be9f1925
commit 22e102ad98
3 changed files with 73 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
tests/quantization/ggml/test_ggml.py
View File

@@ -38,7 +38,8 @@ class GgufIntegrationTests(unittest.TestCase):
imatrix_model_id = "duyntnet/TinyLlama-1.1B-Chat-v1.0-imatrix-GGUF"
mistral_model_id = "TheBloke/Mistral-7B-Instruct-v0.2-GGUF"
qwen2_model_id = "Qwen/Qwen1.5-0.5B-Chat-GGUF"
qwen2_moe_model_id = "RichardErkhov/Qwen_-_Qwen1.5-MoE-A2.7B-Chat-gguf"
qwen2moe_model_id = "gdax/Qwen1.5-MoE-A2.7B_gguf"
qwen2moe_original_model_id = "Qwen/Qwen1.5-MoE-A2.7B"
llama3_model_id = "NousResearch/Meta-Llama-3-8B-GGUF"
tinyllama_model_id = "PenutChen/TinyLlama-1.1B-Chat-v1.0-GGUF"
phi3_model_id = "microsoft/Phi-3-mini-4k-instruct-gguf"
@@ -72,7 +73,7 @@ class GgufIntegrationTests(unittest.TestCase):
q4_0_phi3_model_id = "Phi-3-mini-4k-instruct-q4.gguf"
q4_0_mistral_model_id = "mistral-7b-instruct-v0.2.Q4_0.gguf"
q4_0_qwen2_model_id = "qwen1_5-0_5b-chat-q4_0.gguf"
q4_0_qwen2_moe_model_id = "Qwen1.5-MoE-A2.7B-Chat.Q4_0.gguf"
q8_qwen2moe_model_id = "Qwen1.5-MoE-A2.7B_Q8_0.gguf"
q4_llama3_model_id = "Meta-Llama-3-8B-Q4_K_M.gguf"
fp16_bloom_model_id = "bloom-560m.fp16.gguf"
q8_bloom_model_id = "bloom-560m.q8_0.gguf"
@@ -80,6 +81,7 @@ class GgufIntegrationTests(unittest.TestCase):
q2_k_falcon7b_model_id = "falcon-7b-q2_k.gguf"
fp16_falcon7b_model_id = "falcon-7b-fp16.gguf"
q2_k_falcon40b_model_id = "tiiuae-falcon-40b-Q2_K.gguf"
fp16_qwen2moe_model_id = "Qwen1.5-MoE-A2.7B.gguf"
example_text = "Hello"
@@ -344,21 +346,39 @@ class GgufIntegrationTests(unittest.TestCase):
EXPECTED_TEXT = "Hello.jsoup\n\nI am a beginner"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_qwen2_moe_q4_0(self):
tokenizer = AutoTokenizer.from_pretrained(self.qwen2_moe_model_id, gguf_file=self.q4_0_qwen2_moe_model_id)
def test_qwen2moe_q8(self):
tokenizer = AutoTokenizer.from_pretrained(self.qwen2moe_model_id, gguf_file=self.q8_qwen2moe_model_id)
model = AutoModelForCausalLM.from_pretrained(
self.qwen2_moe_model_id,
gguf_file=self.q4_0_qwen2_moe_model_id,
device_map="auto",
self.qwen2moe_model_id,
gguf_file=self.q8_qwen2moe_model_id,
torch_dtype=torch.float16,
)
text = tokenizer(self.example_text, return_tensors="pt").to(torch_device)
text = tokenizer(self.example_text, return_tensors="pt")
out = model.generate(**text, max_new_tokens=10)
EXPECTED_TEXT = "Hello everyone, I'm a newbie here and would like"
EXPECTED_TEXT = "Hello, I am a 20 year old male"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_qwen2moe_weights_conversion_fp16(self):
quantized_model = AutoModelForCausalLM.from_pretrained(
self.qwen2moe_model_id,
gguf_file=self.fp16_qwen2moe_model_id,
torch_dtype=torch.float16,
)
original_model = AutoModelForCausalLM.from_pretrained(
self.qwen2moe_original_model_id,
torch_dtype=torch.float16,
)
quantized_state_dict = quantized_model.state_dict()
original_state_dict = original_model.state_dict()
for layer_name, original_params in original_state_dict.items():
if layer_name in quantized_state_dict:
self.assertTrue(original_params.shape == quantized_state_dict[layer_name].shape)
torch.testing.assert_close(original_params, quantized_state_dict[layer_name])
def test_phi3_q4_0(self):
tokenizer = AutoTokenizer.from_pretrained(self.phi3_model_id, gguf_file=self.q4_0_phi3_model_id)
model = AutoModelForCausalLM.from_pretrained(
@@ -422,7 +442,7 @@ class GgufIntegrationTests(unittest.TestCase):
text = tokenizer(self.example_text, return_tensors="pt").to(torch_device)
out = model.generate(**text, max_new_tokens=10)
EXPECTED_TEXT = "Hello, I just want to say that I am very"
EXPECTED_TEXT = "Hello, I just want to say that I am just"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_bloom_weights_conversion_fp16(self):