From f7e80721ebab64bc6440a46a3dab6004dc43c973 Mon Sep 17 00:00:00 2001
From: tznurmin <toni.nurminen@gmail.com>
Date: Mon, 7 Sep 2020 10:12:22 +0000
Subject: [PATCH] Fixed the default number of attention heads in Reformer
 Configuration (#6973)

---
 src/transformers/configuration_reformer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/transformers/configuration_reformer.py b/src/transformers/configuration_reformer.py
index 7532684fdd..87c4b9e6fb 100755
--- a/src/transformers/configuration_reformer.py
+++ b/src/transformers/configuration_reformer.py
@@ -160,7 +160,7 @@ class ReformerConfig(PretrainedConfig):
         lsh_num_chunks_before=1,
         lsh_num_chunks_after=0,
         max_position_embeddings=4096,
-        num_attention_heads=2,
+        num_attention_heads=12,
         num_buckets=None,
         num_hashes=1,
         pad_token_id=0,