SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Adafactor: avoid updating group["lr"] attributes (#9751)

Browse Source 
This affects Adafactor with relative_step=False and scale_parameter=True.
Updating group["lr"] makes the result of ._get_lr() depends on the previous call,
i.e., on the scale of other parameters. This isn't supposed to happen.
This commit is contained in:
CeShine Lee
2021-02-01 21:07:33 +08:00
committed by GitHub
parent 115d97dd2f
commit 8672bcda1f
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/optimization.py
View File

@@ -546,7 +546,7 @@ class Adafactor(Optimizer):
state["step"] += 1 state["step"] += 1
state["RMS"] = self._rms(p_data_fp32) state["RMS"] = self._rms(p_data_fp32)
group["lr"] = self._get_lr(group, state) lr = self._get_lr(group, state)
beta2t = 1.0 - math.pow(state["step"], group["decay_rate"]) beta2t = 1.0 - math.pow(state["step"], group["decay_rate"])
update = (grad ** 2) + group["eps"][0] update = (grad ** 2) + group["eps"][0]
@@ -567,7 +567,7 @@ class Adafactor(Optimizer):
update = exp_avg_sq.rsqrt().mul_(grad) update = exp_avg_sq.rsqrt().mul_(grad)
update.div_((self._rms(update) / group["clip_threshold"]).clamp_(min=1.0)) update.div_((self._rms(update) / group["clip_threshold"]).clamp_(min=1.0))
update.mul_(group["lr"]) update.mul_(lr)
if use_first_moment: if use_first_moment:
exp_avg = state["exp_avg"] exp_avg = state["exp_avg"]
@@ -575,7 +575,7 @@ class Adafactor(Optimizer):
update = exp_avg update = exp_avg
if group["weight_decay"] != 0: if group["weight_decay"] != 0:
p_data_fp32.add_(-group["weight_decay"] * group["lr"], p_data_fp32) p_data_fp32.add_(-group["weight_decay"] * lr, p_data_fp32)
p_data_fp32.add_(-update) p_data_fp32.add_(-update)