SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

add a decoder layer for Bert

Browse Source
This commit is contained in:
Rémi Louf
2019-10-08 10:11:02 +02:00
parent a0dcefa382
commit cd6a59d5c1
1 changed files with 21 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
transformers/modeling_bert.py
View File

@@ -341,10 +341,28 @@ class BertEncoderLayer(nn.Module):
class BertDecoderLayer(nn.Module):
def __init__(self, config):
super(BertDecoderLayer, self).__init__()
raise NotImplementedError
self.self_attention = BertAttention(config)
self.attention = BertAttention(config)
self.intermediate = BertIntermediate(config)
self.output = BertOutput(config)
def forward(self, hidden_state, encoder_output):
raise NotImplementedError
def forward(self, hidden_states, encoder_outputs, attention_mask=None, head_mask=None):
self_attention_outputs = self.self_attention(query_tensor=hidden_states,
key_tensor=hidden_states,
value_tensor=hidden_states,
attention_mask=attention_mask,
head_mask=head_mask)
self_attention_output = self_attention_outputs[0]
attention_outputs = self.attention(query_tensor=self_attention_output,
key_tensor=encoder_outputs,
value_tensor=encoder_outputs,
attention_mask=attention_mask,
head_mask=head_mask)
attention_output = attention_outputs[0]
intermediate_output = self.intermediate(attention_output)
layer_output = self.output(intermediate_output, attention_output)
outputs = (layer_output,) + attention_outputs[1:]
return outputs
class BertEncoder(nn.Module):