From 434d15da8ed88a790d5044b3edf3b496bed425a9 Mon Sep 17 00:00:00 2001 From: Dan Hendrycks Date: Sat, 16 Feb 2019 12:17:52 -0800 Subject: [PATCH] Update activation function docstring --- pytorch_pretrained_bert/modeling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_pretrained_bert/modeling.py b/pytorch_pretrained_bert/modeling.py index 05f61b4c76..54ac86d0fc 100644 --- a/pytorch_pretrained_bert/modeling.py +++ b/pytorch_pretrained_bert/modeling.py @@ -114,6 +114,7 @@ def gelu(x): """Implementation of the gelu activation function. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 """ return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))