SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Division to num_train_optimizer of global_step in lr_this_step is removed.

Browse Source
This commit is contained in:
burcturkoglu
2019-05-09 10:51:38 +03:00
parent 3ae8c8be1e
commit 5289b4b9e0
5 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/lm_finetuning/finetune_on_pregenerated.py
View File

@@ -315,7 +315,7 @@ def main():
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step/num_train_optimization_steps,
lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step

2
examples/lm_finetuning/simple_lm_finetuning.py
View File

@@ -603,7 +603,7 @@ def main():
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step/num_train_optimization_steps,
lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step,
args.warmup_proportion)
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step