SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add gguf support for bloom (#33473)

Browse Source 
* add bloom arch support for gguf

* apply format

* small refactoring, bug fix in GGUF_TENSOR_MAPPING naming

* optimize bloom GGUF_TENSOR_MAPPING

* implement reverse reshaping for bloom gguf

* add qkv weights test

* add q_8 test for bloom
This commit is contained in:
Vladislav Bronzov
2024-09-27 12:13:40 +02:00
committed by GitHub
parent 3e039d3827
commit 9d200cfbee
6 changed files with 140 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
tests/quantization/ggml/test_ggml.py
View File

@@ -42,6 +42,8 @@ class GgufIntegrationTests(unittest.TestCase):
llama3_model_id = "NousResearch/Meta-Llama-3-8B-GGUF"
tinyllama_model_id = "PenutChen/TinyLlama-1.1B-Chat-v1.0-GGUF"
phi3_model_id = "microsoft/Phi-3-mini-4k-instruct-gguf"
bloom_model_id = "afrideva/bloom-560m-GGUF"
original_bloom_model_id = "bigscience/bloom-560m"
# standard quants
q4_0_gguf_model_id = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
@@ -69,6 +71,8 @@ class GgufIntegrationTests(unittest.TestCase):
q4_0_qwen2_model_id = "qwen1_5-0_5b-chat-q4_0.gguf"
q4_0_qwen2_moe_model_id = "Qwen1.5-MoE-A2.7B-Chat.Q4_0.gguf"
q4_llama3_model_id = "Meta-Llama-3-8B-Q4_K_M.gguf"
fp16_bloom_model_id = "bloom-560m.fp16.gguf"
q8_bloom_model_id = "bloom-560m.q8_0.gguf"
f16_tinyllama_model_id = "TinyLlama-1.1B-Chat-v1.0.FP16.gguf"
example_text = "Hello"
@@ -385,6 +389,62 @@ class GgufIntegrationTests(unittest.TestCase):
EXPECTED_TEXT = "Hello, I am interested in [The Park]\nThe"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_bloom_fp16(self):
tokenizer = AutoTokenizer.from_pretrained(self.bloom_model_id, gguf_file=self.fp16_bloom_model_id)
model = AutoModelForCausalLM.from_pretrained(
self.bloom_model_id,
gguf_file=self.fp16_bloom_model_id,
device_map="auto",
torch_dtype=torch.float16,
)
text = tokenizer(self.example_text, return_tensors="pt").to(torch_device)
out = model.generate(**text, max_new_tokens=10)
EXPECTED_TEXT = "Hello, I just want to say that I am very"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_bloom_q8_0(self):
tokenizer = AutoTokenizer.from_pretrained(self.bloom_model_id, gguf_file=self.q8_bloom_model_id)
model = AutoModelForCausalLM.from_pretrained(
self.bloom_model_id,
gguf_file=self.q8_bloom_model_id,
device_map="auto",
torch_dtype=torch.float16,
)
text = tokenizer(self.example_text, return_tensors="pt").to(torch_device)
out = model.generate(**text, max_new_tokens=10)
EXPECTED_TEXT = "Hello, I just want to say that I am very"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_bloom_weights_conversion_fp16(self):
quantized_model = AutoModelForCausalLM.from_pretrained(
self.bloom_model_id,
gguf_file=self.fp16_bloom_model_id,
device_map="auto",
torch_dtype=torch.float16,
)
original_model = AutoModelForCausalLM.from_pretrained(
self.original_bloom_model_id,
device_map="auto",
torch_dtype=torch.float16,
)
quantized_state_dict = quantized_model.state_dict()
original_state_dict = original_model.state_dict()
for (quantized_name, quantized_param), (original_name, original_param) in zip(
quantized_state_dict.items(), original_state_dict.items()
):
if (
"self_attention.query_key_value" in quantized_name
and "self_attention.query_key_value" in original_name
):
self.assertTrue(quantized_param.shape == original_param.shape)
torch.testing.assert_close(quantized_param, original_param)
def test_tokenization_xnli(self):
import tqdm
from datasets import load_dataset