From e6d106a01be43d66fddcece53e80139d90b2f653 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Sat, 3 Nov 2018 10:47:10 +0100 Subject: [PATCH] comment on gelu function --- modeling_pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modeling_pytorch.py b/modeling_pytorch.py index e857be05fa..1e25a978b3 100644 --- a/modeling_pytorch.py +++ b/modeling_pytorch.py @@ -28,8 +28,8 @@ from torch.nn import CrossEntropyLoss def gelu(x): return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) - # OpenAI GPT gelu version : - # return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + # For information: OpenAI GPT gelu version is a bit different: + # 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) class BertConfig(object):