SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #495 from SudoSharma/patch-2

Browse Source 
Fix gradient overflow issue during attention mask
This commit is contained in:
Thomas Wolf
2019-04-17 11:10:36 +02:00
committed by GitHub
parent 46078e1b46 9e666aaa29
commit 2e153930cf
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytorch_pretrained_bert/modeling_gpt2.py
View File

@@ -218,7 +218,7 @@ class Attention(nn.Module):
w = w / math.sqrt(v.size(-1)) w = w / math.sqrt(v.size(-1))
nd, ns = w.size(-2), w.size(-1) nd, ns = w.size(-2), w.size(-1)
b = self.bias[:, :, ns-nd:ns, :ns] b = self.bias[:, :, ns-nd:ns, :ns]
w = w * b - 1e10 * (1 - b) w = w * b - 1e4 * (1 - b)
w = nn.Softmax(dim=-1)(w) w = nn.Softmax(dim=-1)(w)
return torch.matmul(w, v) return torch.matmul(w, v)