SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fixing param.grad is None in fp16 examples

Browse Source
This commit is contained in:
thomwolf
2018-11-20 10:01:21 +01:00
parent da73925f6a
commit 2f21497d3e
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/run_classifier.py
View File

@@ -555,7 +555,8 @@ def main():
if args.fp16 and args.loss_scale != 1.0: if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training # scale down gradients for fp16 training
for param in model.parameters(): for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan: if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")

3
examples/run_squad.py
View File

@@ -898,7 +898,8 @@ def main():
if args.fp16 and args.loss_scale != 1.0: if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training # scale down gradients for fp16 training
for param in model.parameters(): for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan: if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")