SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #171 from donglixp/patch-1

Browse Source 
LayerNorm initialization
This commit is contained in:
Thomas Wolf
2019-01-07 12:44:46 +01:00
committed by GitHub
parent d3d56f9a0b d0d9b384f2
commit e048c7f1c8
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pytorch_pretrained_bert/modeling.py
View File

@@ -439,8 +439,8 @@ class PreTrainedBertModel(nn.Module):
# cf https://github.com/pytorch/pytorch/pull/5617 # cf https://github.com/pytorch/pytorch/pull/5617
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
elif isinstance(module, BertLayerNorm): elif isinstance(module, BertLayerNorm):
module.bias.data.normal_(mean=0.0, std=self.config.initializer_range) module.bias.data.zero_()
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) module.weight.data.fill_(1.0)
if isinstance(module, nn.Linear) and module.bias is not None: if isinstance(module, nn.Linear) and module.bias is not None:
module.bias.data.zero_() module.bias.data.zero_()