SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

change bnb tests (#34713)

Browse Source 
* fix training tests

* fix xpu check

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* rm pdb

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* fix 4bit logits check

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* fix 4bit logits check

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* add xpu check on int8 training

* fix training tests

* add llama test on bnb

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* only cpu and xpu disable autocast training

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* fix format

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

---------

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>
Co-authored-by: Titus <9048635+Titus-von-Koeller@users.noreply.github.com>
This commit is contained in:
jiqing-feng
2024-12-18 22:49:59 +08:00
committed by GitHub
parent da334bcfa8
commit 69e31eb1bf
2 changed files with 67 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
tests/quantization/bnb/test_mixed_int8.py
View File

@@ -48,6 +48,8 @@ from transformers.testing_utils import (
def get_some_linear_layer(model):
if model.config.model_type == "gpt2":
return model.transformer.h[0].mlp.c_fc
elif model.config.model_type == "llama":
return model.model.layers[0].mlp.gate_proj
return model.transformer.h[0].mlp.dense_4h_to_h
@@ -65,12 +67,12 @@ if is_torch_available():
class LoRALayer(nn.Module):
"""Wraps a linear layer with LoRA-like adapter - Used for testing purposes only"""
def __init__(self, module: nn.Module, rank: int):
def __init__(self, module: nn.Module, rank: int, dtype: torch.dtype):
super().__init__()
self.module = module
self.adapter = nn.Sequential(
nn.Linear(module.in_features, rank, bias=False),
nn.Linear(rank, module.out_features, bias=False),
nn.Linear(module.in_features, rank, bias=False, dtype=dtype),
nn.Linear(rank, module.out_features, bias=False, dtype=dtype),
)
small_std = (2.0 / (5 * min(module.in_features, module.out_features))) ** 0.5
nn.init.normal_(self.adapter[0].weight, std=small_std)
@@ -858,29 +860,36 @@ class MixedInt8TestTraining(BaseMixedInt8Test):
if torch.cuda.is_available():
self.assertEqual(set(model.hf_device_map.values()), {torch.cuda.current_device()})
elif torch.xpu.is_available():
self.assertEqual(set(model.hf_device_map.values()), {f"xpu:{torch.xpu.current_device()}"})
else:
self.assertTrue(all(param.device.type == "cpu" for param in model.parameters()))
for param in model.parameters():
param.requires_grad = False # freeze the model - train adapters later
if param.ndim == 1:
# cast the small parameters (e.g. layernorm) to fp32 for stability
# cast all non INT8 parameters to fp32
if param.dtype in (torch.float16, torch.bfloat16) and param.__class__.__name__ != "Params4bit":
param.data = param.data.to(torch.float32)
# Step 2: add adapters
for _, module in model.named_modules():
if isinstance(module, OPTAttention):
module.q_proj = LoRALayer(module.q_proj, rank=16)
module.k_proj = LoRALayer(module.k_proj, rank=16)
module.v_proj = LoRALayer(module.v_proj, rank=16)
module.q_proj = LoRALayer(module.q_proj, rank=16, dtype=model.dtype)
module.k_proj = LoRALayer(module.k_proj, rank=16, dtype=model.dtype)
module.v_proj = LoRALayer(module.v_proj, rank=16, dtype=model.dtype)
# Step 3: dummy batch
batch = self.tokenizer("Test batch ", return_tensors="pt").to(torch_device)
# Step 4: Check if the gradient is not None
with torch.autocast(torch_device):
if torch_device in {"xpu", "cpu"}:
# XPU and CPU finetune do not support autocast for now.
out = model.forward(**batch)
out.logits.norm().backward()
else:
with torch.autocast(torch_device):
out = model.forward(**batch)
out.logits.norm().backward()
for module in model.modules():
if isinstance(module, LoRALayer):
@@ -891,6 +900,7 @@ class MixedInt8TestTraining(BaseMixedInt8Test):
@apply_skip_if_not_implemented
@unittest.skipIf(torch_device == "xpu", reason="XPU has precision issue on gpt model, will test it once fixed")
class MixedInt8GPT2Test(MixedInt8Test):
model_name = "openai-community/gpt2-xl"
EXPECTED_RELATIVE_DIFFERENCE = 1.8720077507258357
@@ -922,3 +932,30 @@ class MixedInt8GPT2Test(MixedInt8Test):
output_sequences = model.generate(input_ids=encoded_input["input_ids"].to(torch_device), max_new_tokens=10)
self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS)
class MixedInt8LlamaTest(MixedInt8Test):
model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
EXPECTED_RELATIVE_DIFFERENCE = 1.7869331026479096
EXPECTED_OUTPUTS = set()
EXPECTED_OUTPUTS.add("Hello my name is John Smith and I am a software engineer. I")
def test_int8_from_pretrained(self):
r"""
Test whether loading a 8bit model from the Hub works as expected
"""
from bitsandbytes.nn import Int8Params
model_id = "Jiqing/TinyLlama-1.1B-Chat-v1.0-bnb-8bit"
model = AutoModelForCausalLM.from_pretrained(model_id)
linear = get_some_linear_layer(model)
self.assertTrue(linear.weight.__class__ == Int8Params)
self.assertTrue(hasattr(linear.weight, "SCB"))
# generate
encoded_input = self.tokenizer(self.input_text, return_tensors="pt")
output_sequences = model.generate(input_ids=encoded_input["input_ids"].to(torch_device), max_new_tokens=10)
self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS)