SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add Gemma2 GGUF support (#34002)

Browse Source 
* initial setup for ggml.py

* initial setup of GGUFGemma2Converter class

* Add gemma2 model to gguf.md doc

* Partial work on GGUF_TENSOR_MAPPING

* initial setup of GGUF_TENSOR_MAPPING for Gemma2

* refactor: rename GemmaConvert class to GemmaConverter for naming consistency

* feat: complete gemma2 tensor mapping implementation

* feat: add initial implementation of GGUFGemmaConverter

* feat: complete GGUFGemmaConverter implementation

* feat: add test code for gemma2

* refactor: minor code cleanup

* refactor: minor code cleanup

* fix: resolve suggestions

* Update tests/quantization/ggml/test_ggml.py

Co-authored-by: Isotr0py <2037008807@qq.com>

---------

Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
Yijun Lee
2025-01-03 22:50:07 +09:00
committed by GitHub
parent 1fe2d53d4e
commit e5fd865eba
5 changed files with 180 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
tests/quantization/ggml/test_ggml.py
View File

@@ -64,6 +64,8 @@ class GgufIntegrationTests(unittest.TestCase):
mamba_model_id = "jpodivin/mamba-2.8b-hf-GGUF"
nemotron_original_model_id = "nvidia/Nemotron-Mini-4B-Instruct"
nemotron_model_id = "bartowski/Nemotron-Mini-4B-Instruct-GGUF"
original_gemma2_model_id = "google/gemma-2-2b-it"
gemma2_model_id = "bartowski/gemma-2-2b-it-GGUF"
# standard quants
q4_0_gguf_model_id = "tinyllama-1.1b-chat-v1.0.Q4_0.gguf"
@@ -111,6 +113,9 @@ class GgufIntegrationTests(unittest.TestCase):
fp16_mamba_model_id = "ggml-model-f16.gguf"
q6_k_nemotron_model_id = "Nemotron-Mini-4B-Instruct-Q6_K.gguf"
fp16_nemotron_model_id = "Nemotron-Mini-4B-Instruct-f16.gguf"
q3_k_gemma2_model_id = "gemma-2-2b-it-Q3_K_L.gguf"
q8_0_gemma2_model_id = "gemma-2-2b-it-Q8_0.gguf"
fp32_gemma2_model_id = "gemma-2-2b-it-f32.gguf"
example_text = "Hello"
@@ -833,6 +838,70 @@ class GgufIntegrationTests(unittest.TestCase):
EXPECTED_TEXT = "'Hello. hotmail.com.'"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_gemma2_q3_k(self):
model = AutoModelForCausalLM.from_pretrained(
self.gemma2_model_id,
gguf_file=self.q3_k_gemma2_model_id,
torch_dtype=torch.float16,
)
tokenizer = AutoTokenizer.from_pretrained(self.gemma2_model_id, gguf_file=self.q3_k_gemma2_model_id)
text = tokenizer(self.example_text, return_tensors="pt")["input_ids"]
out = model.generate(text, max_new_tokens=10)
EXPECTED_TEXT = "Hello! 👋\n\nI'm trying to create a"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_gemma2_q8_0(self):
model = AutoModelForCausalLM.from_pretrained(
self.gemma2_model_id,
gguf_file=self.q8_0_gemma2_model_id,
torch_dtype=torch.float16,
)
tokenizer = AutoTokenizer.from_pretrained(self.gemma2_model_id, gguf_file=self.q8_0_gemma2_model_id)
text = tokenizer(self.example_text, return_tensors="pt")["input_ids"]
out = model.generate(text, max_new_tokens=10)
EXPECTED_TEXT = "Hello! 👋\n\nI'm a large language model"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_gemma2_fp32(self):
model = AutoModelForCausalLM.from_pretrained(
self.gemma2_model_id,
gguf_file=self.fp32_gemma2_model_id,
torch_dtype=torch.float16,
)
tokenizer = AutoTokenizer.from_pretrained(self.gemma2_model_id, gguf_file=self.fp32_gemma2_model_id)
text = tokenizer(self.example_text, return_tensors="pt")["input_ids"]
out = model.generate(text, max_new_tokens=10)
EXPECTED_TEXT = "Hello! 👋\n\nI'm a large language model"
self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
def test_gemma2_weights_conversion_fp32(self):
original_model = AutoModelForCausalLM.from_pretrained(
self.original_gemma2_model_id,
torch_dtype=torch.float16,
)
converted_model = AutoModelForCausalLM.from_pretrained(
self.gemma2_model_id,
gguf_file=self.fp32_gemma2_model_id,
torch_dtype=torch.float16,
)
converted_state_dict = converted_model.state_dict()
original_state_dict = original_model.state_dict()
for layer_name, original_params in original_state_dict.items():
if layer_name in converted_state_dict:
self.assertTrue(original_params.shape == converted_state_dict[layer_name].shape)
torch.testing.assert_close(original_params, converted_state_dict[layer_name])
else:
raise ValueError(f"Layer {layer_name} is not presented in GGUF model")
def test_tokenization_xnli(self):
import tqdm
from datasets import load_dataset