SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fixing param.grad is None in fp16 examples

Browse Source
This commit is contained in:
thomwolf
2018-11-20 10:01:21 +01:00
parent da73925f6a
commit 2f21497d3e
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/run_classifier.py
View File

@@ -555,6 +555,7 @@ def main():
if args.fp16 and args.loss_scale != 1.0: if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training # scale down gradients for fp16 training
for param in model.parameters(): for param in model.parameters():
if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan: if is_nan:

1
examples/run_squad.py
View File

@@ -898,6 +898,7 @@ def main():
if args.fp16 and args.loss_scale != 1.0: if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training # scale down gradients for fp16 training
for param in model.parameters(): for param in model.parameters():
if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan: if is_nan: