SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

comment on gelu function

Browse Source
This commit is contained in:
thomwolf
2018-11-03 10:47:10 +01:00
parent 01b1a05343
commit e6d106a01b
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modeling_pytorch.py
View File

@@ -28,8 +28,8 @@ from torch.nn import CrossEntropyLoss
def gelu(x): def gelu(x):
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
# OpenAI GPT gelu version : # For information: OpenAI GPT gelu version is a bit different:
# return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) # 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
class BertConfig(object): class BertConfig(object):