SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix two bugs with --logging_first_step (#8193)

Browse Source 
* make sure that logging_first_step evaluates

* fix bug with incorrect loss on logging_first_step

* fix style

* logging_first_step only logs, not evals
This commit is contained in:
Abi See
2020-10-30 13:45:38 -07:00
committed by GitHub
parent 689ff74f99
commit 8f1c960ee7
2 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/trainer.py
View File

@@ -729,6 +729,7 @@ class Trainer:
tr_loss = torch.tensor(0.0).to(self.args.device) tr_loss = torch.tensor(0.0).to(self.args.device)
self._logging_loss_scalar = 0 self._logging_loss_scalar = 0
self._globalstep_last_logged = 0
self._total_flos = self.state.total_flos self._total_flos = self.state.total_flos
model.zero_grad() model.zero_grad()
@@ -849,7 +850,9 @@ class Trainer:
if self.control.should_log: if self.control.should_log:
logs: Dict[str, float] = {} logs: Dict[str, float] = {}
tr_loss_scalar = tr_loss.item() tr_loss_scalar = tr_loss.item()
logs["loss"] = (tr_loss_scalar - self._logging_loss_scalar) / self.args.logging_steps logs["loss"] = (tr_loss_scalar - self._logging_loss_scalar) / (
self.state.global_step - self._globalstep_last_logged
)
# backward compatibility for pytorch schedulers # backward compatibility for pytorch schedulers
logs["learning_rate"] = ( logs["learning_rate"] = (
self.lr_scheduler.get_last_lr()[0] self.lr_scheduler.get_last_lr()[0]
@@ -857,6 +860,7 @@ class Trainer:
else self.lr_scheduler.get_lr()[0] else self.lr_scheduler.get_lr()[0]
) )
self._logging_loss_scalar = tr_loss_scalar self._logging_loss_scalar = tr_loss_scalar
self._globalstep_last_logged = self.state.global_step
self.log(logs) self.log(logs)

2
src/transformers/training_args.py
View File

@@ -250,7 +250,7 @@ class TrainingArguments:
warmup_steps: int = field(default=0, metadata={"help": "Linear warmup over warmup_steps."}) warmup_steps: int = field(default=0, metadata={"help": "Linear warmup over warmup_steps."})
logging_dir: Optional[str] = field(default_factory=default_logdir, metadata={"help": "Tensorboard log dir."}) logging_dir: Optional[str] = field(default_factory=default_logdir, metadata={"help": "Tensorboard log dir."})
logging_first_step: bool = field(default=False, metadata={"help": "Log and eval the first global_step"}) logging_first_step: bool = field(default=False, metadata={"help": "Log the first global_step"})
logging_steps: int = field(default=500, metadata={"help": "Log every X updates steps."}) logging_steps: int = field(default=500, metadata={"help": "Log every X updates steps."})
save_steps: int = field(default=500, metadata={"help": "Save checkpoint every X updates steps."}) save_steps: int = field(default=500, metadata={"help": "Save checkpoint every X updates steps."})
save_total_limit: Optional[int] = field( save_total_limit: Optional[int] = field(