SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add gguf support for gpt2 (#34044)

Browse Source 
* add gpt2 gguf support

* add doc change

* small refactoring
This commit is contained in:
Vladislav Bronzov
2024-10-10 13:42:18 +02:00
committed by GitHub
parent 66e08dba71
commit c9afee5392
5 changed files with 95 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
tests/quantization/ggml/test_ggml.py
View File

@@ -51,6 +51,9 @@ class GgufIntegrationTests(unittest.TestCase):
stablelm_model_id = "afrideva/stablelm-3b-4e1t-GGUF"
stablelm2_model_id = "afrideva/stablelm-2-1_6b-GGUF"
original_stablelm2_model_id = "stabilityai/stablelm-2-1_6b"
gpt2_model_id = "mradermacher/gpt2-GGUF"
gpt2_original_model_id = "openai-community/gpt2"
gpt2_xl_model_id = "RichardErkhov/openai-community_-_gpt2-xl-gguf"
# standard quants
q4_0_gguf_model_id = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
@@ -87,6 +90,9 @@ class GgufIntegrationTests(unittest.TestCase):
fp16_falcon7b_model_id = "falcon-7b-fp16.gguf"
q2_k_falcon40b_model_id = "tiiuae-falcon-40b-Q2_K.gguf"
fp16_qwen2moe_model_id = "Qwen1.5-MoE-A2.7B.gguf"
fp16_gpt2_model_id = "gpt2.f16.gguf"
q8_gpt2_model_id = "gpt2.Q8_0.gguf"
q6_k_gpt2_xl_model_id = "gpt2-xl.Q6_K.gguf"
example_text = "Hello"
@@ -476,6 +482,53 @@ class GgufIntegrationTests(unittest.TestCase):
self.assertTrue(quantized_param.shape == original_param.shape)
torch.testing.assert_close(quantized_param, original_param)
def test_gpt2_q8(self):
tokenizer = AutoTokenizer.from_pretrained(self.gpt2_model_id, gguf_file=self.q8_gpt2_model_id)
model = AutoModelForCausalLM.from_pretrained(
self.gpt2_model_id,
gguf_file=self.q8_gpt2_model_id,
torch_dtype=torch.float16,
)
text = tokenizer(self.example_text, return_tensors="pt")
out = model.generate(**text, max_new_tokens=10)
EXPECTED_TEXT = "Hello, I'm sorry. I'm sorry. I"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_gpt2_weights_conversion_fp16(self):
quantized_model = AutoModelForCausalLM.from_pretrained(
self.gpt2_model_id,
gguf_file=self.fp16_gpt2_model_id,
torch_dtype=torch.float16,
)
original_model = AutoModelForCausalLM.from_pretrained(
self.gpt2_original_model_id,
torch_dtype=torch.float16,
)
quantized_state_dict = quantized_model.state_dict()
original_state_dict = original_model.state_dict()
for layer_name, original_params in original_state_dict.items():
if layer_name in quantized_state_dict:
self.assertTrue(original_params.shape == quantized_state_dict[layer_name].shape)
torch.testing.assert_close(original_params, quantized_state_dict[layer_name])
def test_gpt2_xl_Q6_K(self):
tokenizer = AutoTokenizer.from_pretrained(self.gpt2_xl_model_id, gguf_file=self.q6_k_gpt2_xl_model_id)
model = AutoModelForCausalLM.from_pretrained(
self.gpt2_xl_model_id,
gguf_file=self.q6_k_gpt2_xl_model_id,
torch_dtype=torch.float16,
)
text = tokenizer(self.example_text, return_tensors="pt")
out = model.generate(**text, max_new_tokens=10)
EXPECTED_TEXT = "Hello, I'm a newbie to the world of"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
@unittest.skip(reason="Heavy memory")
def test_falcon40b_q2_k(self):
tokenizer = AutoTokenizer.from_pretrained(self.falcon40b_model_id, gguf_file=self.q2_k_falcon40b_model_id)