From 60a372387f036ad329c703ab0956e50fc9f2fa1a Mon Sep 17 00:00:00 2001 From: lukovnikov Date: Wed, 27 Feb 2019 15:54:09 +0100 Subject: [PATCH] added warning --- pytorch_pretrained_bert/optimization.py | 5 +++++ pytorch_pretrained_bert/optimization_openai.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pytorch_pretrained_bert/optimization.py b/pytorch_pretrained_bert/optimization.py index 7cde422453..6681410feb 100644 --- a/pytorch_pretrained_bert/optimization.py +++ b/pytorch_pretrained_bert/optimization.py @@ -19,6 +19,9 @@ import torch from torch.optim import Optimizer from torch.optim.optimizer import required from torch.nn.utils import clip_grad_norm_ +import logging + +logger = logging.getLogger(__name__) def warmup_cosine(x, warmup=0.002): if x < warmup: @@ -37,6 +40,8 @@ def warmup_linear(x, warmup=0.002): After `t_total`-th training step, learning rate is zero. """ if x < warmup: return x/warmup + if x > 1: + logger.warning("Training beyond specified 't_total' steps. Learning rate set to zero. Please set 't_total' of BertAdam correctly.") return max((x-1.)/(warmup-1.), 0) SCHEDULES = { diff --git a/pytorch_pretrained_bert/optimization_openai.py b/pytorch_pretrained_bert/optimization_openai.py index 6cdc6c8f0c..7df6369023 100644 --- a/pytorch_pretrained_bert/optimization_openai.py +++ b/pytorch_pretrained_bert/optimization_openai.py @@ -19,6 +19,9 @@ import torch from torch.optim import Optimizer from torch.optim.optimizer import required from torch.nn.utils import clip_grad_norm_ +import logging + +logger = logging.getLogger(__name__) def warmup_cosine(x, warmup=0.002): if x < warmup: @@ -37,6 +40,8 @@ def warmup_linear(x, warmup=0.002): After `t_total`-th training step, learning rate is zero. """ if x < warmup: return x/warmup + if x > 1: + logger.warning("Training beyond specified 't_total' steps. Learning rate set to zero. Please set 't_total' of BertAdam correctly.") return max((x-1.)/(warmup-1.), 0) SCHEDULES = {