SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix use_parallel_residual and qkv_bias for StableLM GGUF config extraction (#34450)

Browse Source 
* fix stablelm qkv_bias

* fix stablelm qkv_bias and use_parallel_residual

* remove original_model.config for stablelm gguf test
This commit is contained in:
Isotr0py
2024-11-06 01:26:20 +08:00
committed by GitHub
parent 9f28d0c5d0
commit e83aaaa86b
2 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/quantization/ggml/test_ggml.py
View File

@@ -673,10 +673,6 @@ class GgufIntegrationTests(unittest.TestCase):
self.stablelm2_model_id,
gguf_file=self.fp16_stablelm2_model_id,
torch_dtype=torch.float16,
# for precise comparison it is required to use the original model config
# as quantized one is different in parameters: use_parallel_residual and use_qkv_bias
# and it highly influences on the output results
config=original_model.config,
)
tokenizer = AutoTokenizer.from_pretrained(self.stablelm2_model_id, gguf_file=self.fp16_stablelm2_model_id)
@@ -703,10 +699,6 @@ class GgufIntegrationTests(unittest.TestCase):
gguf_file=self.fp16_stablelm2_model_id,
device_map="auto",
torch_dtype=torch.float16,
# for precise comparison it is required to use the original model config
# as quantized one is different in parameters: use_parallel_residual and use_qkv_bias
# and it highly influences on the output results
config=original_model.config,
)
converted_state_dict = converted_model.state_dict()