SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix: one more try

Browse Source
This commit is contained in:
Julien Chaumond
2020-05-15 17:38:48 -04:00
parent 55bda52555
commit fc2a4c88ce
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/optimization.py
View File

@@ -152,7 +152,7 @@ class AdamW(Optimizer):
# Decay the first and second moment running average coefficient # Decay the first and second moment running average coefficient
# In-place operations to update the averages at the same time # In-place operations to update the averages at the same time
exp_avg.mul_(beta1).add_(grad, 1.0 - beta1) exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1)
exp_avg_sq.mul_(beta2).addcmul_(grad, grad, 1.0 - beta2) exp_avg_sq.mul_(beta2).addcmul_(grad, grad, 1.0 - beta2)
denom = exp_avg_sq.sqrt().add_(group["eps"]) denom = exp_avg_sq.sqrt().add_(group["eps"])
@@ -173,6 +173,6 @@ class AdamW(Optimizer):
# of the weights to the loss with plain (non-momentum) SGD. # of the weights to the loss with plain (non-momentum) SGD.
# Add weight decay at the end (fixed version) # Add weight decay at the end (fixed version)
if group["weight_decay"] > 0.0: if group["weight_decay"] > 0.0:
p.data.add_(p.data, -group["lr"] * group["weight_decay"]) p.data.add_(p.data, alpha=-group["lr"] * group["weight_decay"])
return loss return loss