diff --git a/setup.py b/setup.py
index e689af6b5f..54d46a289d 100644
--- a/setup.py
+++ b/setup.py
@@ -105,6 +105,7 @@ _deps = [
     "datasets",
     "deepspeed>=0.6.5",
     "dill<0.3.5",
+    "evaluate",
     "fairscale>0.3",
     "faiss-cpu",
     "fastapi",
@@ -289,6 +290,7 @@ extras["testing"] = (
         "psutil",
         "datasets",
         "dill",
+        "evaluate",
         "pytest-timeout",
         "black",
         "sacrebleu",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index b22076588c..a707d0bd85 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -11,6 +11,7 @@ deps = {
     "datasets": "datasets",
     "deepspeed": "deepspeed>=0.6.5",
     "dill": "dill<0.3.5",
+    "evaluate": "evaluate",
     "fairscale": "fairscale>0.3",
     "faiss-cpu": "faiss-cpu",
     "fastapi": "fastapi",