From a6a6d9e6382961dc92a1a08d1bab05a52dc815f9 Mon Sep 17 00:00:00 2001
From: Ikuya Yamada <ikuya@ikuya.net>
Date: Thu, 12 Sep 2019 12:13:37 -1000
Subject: [PATCH] fix padding_idx of RoBERTa model

---
 transformers/modeling_roberta.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py
index 04ffbecc16..7e130a8c52 100644
--- a/transformers/modeling_roberta.py
+++ b/transformers/modeling_roberta.py
@@ -43,6 +43,9 @@ class RobertaEmbeddings(BertEmbeddings):
     def __init__(self, config):
         super(RobertaEmbeddings, self).__init__(config)
         self.padding_idx = 1
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size,
+                                                padding_idx=self.padding_idx)
 
     def forward(self, input_ids, token_type_ids=None, position_ids=None):
         seq_length = input_ids.size(1)