From 77b75d2c78757ab5d9b198b574a7c1ec05dd3219 Mon Sep 17 00:00:00 2001
From: mneilly-et <55827703+mneilly-et@users.noreply.github.com>
Date: Thu, 23 Apr 2020 12:25:31 -0600
Subject: [PATCH] Fix for #3873 to change type of exponent parameter for
 torch.pow() call from int to float (#3924)

---
 src/transformers/activations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/transformers/activations.py b/src/transformers/activations.py
index 86882bb72b..2028e9c7d8 100644
--- a/src/transformers/activations.py
+++ b/src/transformers/activations.py
@@ -26,7 +26,7 @@ def gelu_new(x):
     """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
         Also see https://arxiv.org/abs/1606.08415
     """
-    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
 
 
 if torch.__version__ < "1.4.0":