From 1d438f15b36b8d4f522a543211854f19a8b91cc2 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 30 Aug 2019 20:20:15 -0400 Subject: [PATCH] [XLNet] Use pytorch's layernorm like in BERT See #1089 cc @thomwolf @lysandrejik Also @dhpollack --- pytorch_transformers/modeling_xlnet.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/pytorch_transformers/modeling_xlnet.py b/pytorch_transformers/modeling_xlnet.py index ebf8c1fd63..cc9c1379a1 100644 --- a/pytorch_transformers/modeling_xlnet.py +++ b/pytorch_transformers/modeling_xlnet.py @@ -337,20 +337,7 @@ try: from apex.normalization.fused_layer_norm import FusedLayerNorm as XLNetLayerNorm except (ImportError, AttributeError) as e: logger.info("Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .") - class XLNetLayerNorm(nn.Module): - def __init__(self, d_model, eps=1e-12): - """Construct a layernorm module in the TF style (epsilon inside the square root). - """ - super(XLNetLayerNorm, self).__init__() - self.weight = nn.Parameter(torch.ones(d_model)) - self.bias = nn.Parameter(torch.zeros(d_model)) - self.variance_epsilon = eps - - def forward(self, x): - u = x.mean(-1, keepdim=True) - s = (x - u).pow(2).mean(-1, keepdim=True) - x = (x - u) / torch.sqrt(s + self.variance_epsilon) - return self.weight * x + self.bias + from torch.nn import LayerNorm as XLNetLayerNorm class XLNetRelativeAttention(nn.Module): def __init__(self, config):