SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix gradient clipping for Sharded DDP (#9168)

Browse Source 
* Fix gradient clipping for Sharded DDP

* Fix typos in comments
This commit is contained in:
Sylvain Gugger
2020-12-17 09:44:24 -05:00
committed by GitHub
parent 1aca3d6afa
commit 77d6941e64
1 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/transformers/trainer.py
View File

@@ -804,14 +804,23 @@ class Trainer:
steps_in_epoch <= self.args.gradient_accumulation_steps steps_in_epoch <= self.args.gradient_accumulation_steps
and (step + 1) == steps_in_epoch and (step + 1) == steps_in_epoch
): ):
if self.use_amp: # Gradient clipping
self.scaler.unscale_(self.optimizer) if self.args.max_grad_norm is not None and self.args.max_grad_norm > 0:
torch.nn.utils.clip_grad_norm_(model.parameters(), self.args.max_grad_norm) if self.use_amp:
elif self.use_apex: # AMP: gradients need unscaling
torch.nn.utils.clip_grad_norm_(amp.master_params(self.optimizer), self.args.max_grad_norm) self.scaler.unscale_(self.optimizer)
else:
torch.nn.utils.clip_grad_norm_(model.parameters(), self.args.max_grad_norm)
if hasattr(self.optimizer, "clip_grad_norm"):
# Some optimizers (like the sharded optimizer) have a specific way to do gradient clipping
self.optimizer.clip_grad_norm(self.args.max_grad_norm)
else:
# Revert to normal clipping otherwise, handling Apex or full precision
torch.nn.utils.clip_grad_norm_(
amp.master_params(self.optimizer) if self.use_apex else model.parameters(),
self.args.max_grad_norm,
)
# Optimizer step
if is_torch_tpu_available(): if is_torch_tpu_available():
xm.optimizer_step(self.optimizer) xm.optimizer_step(self.optimizer)
elif self.use_amp: elif self.use_amp: