From 2f21497d3e6e87274e52cd6ae92b224e41371bf6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 20 Nov 2018 10:01:21 +0100 Subject: [PATCH] fixing param.grad is None in fp16 examples --- examples/run_classifier.py | 3 ++- examples/run_squad.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/run_classifier.py b/examples/run_classifier.py index 75692ef39e..990b390f92 100644 --- a/examples/run_classifier.py +++ b/examples/run_classifier.py @@ -555,7 +555,8 @@ def main(): if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): - param.grad.data = param.grad.data / args.loss_scale + if param.grad is not None: + param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") diff --git a/examples/run_squad.py b/examples/run_squad.py index 3b54636207..b166757656 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -898,7 +898,8 @@ def main(): if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): - param.grad.data = param.grad.data / args.loss_scale + if param.grad is not None: + param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")