Fixed the default number of attention heads in Reformer Configuration (#6973)

2020-09-07 10:12:22 +00:00
parent e20d8895bd
commit f7e80721eb
1 changed files with 1 additions and 1 deletions
--- a/src/transformers/configuration_reformer.py
+++ b/src/transformers/configuration_reformer.py
@@ -160,7 +160,7 @@ class ReformerConfig(PretrainedConfig):
        lsh_num_chunks_before=1,
        lsh_num_chunks_after=0,
        max_position_embeddings=4096,
-        num_attention_heads=2,
+        num_attention_heads=12,
        num_buckets=None,
        num_hashes=1,
        pad_token_id=0,