diff --git a/setup.py b/setup.py
index 43b33be417..f553e69bc1 100644
--- a/setup.py
+++ b/setup.py
@@ -151,7 +151,7 @@ _deps = [
     "psutil",
     "pyyaml>=5.1",
     "pydantic",
-    "pytest",
+    "pytest>=7.2.0",
     "pytest-timeout",
     "pytest-xdist",
     "python>=3.7.0",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index e79a11ba54..4819b959cb 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -52,7 +52,7 @@ deps = {
     "psutil": "psutil",
     "pyyaml": "pyyaml>=5.1",
     "pydantic": "pydantic",
-    "pytest": "pytest",
+    "pytest": "pytest>=7.2.0",
     "pytest-timeout": "pytest-timeout",
     "pytest-xdist": "pytest-xdist",
     "python": "python>=3.7.0",
diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py
index 8aa1015f39..9cb6dd4f8b 100644
--- a/utils/tests_fetcher.py
+++ b/utils/tests_fetcher.py
@@ -225,10 +225,23 @@ def get_diff_for_py_and_mdx_files(repo, base_commit, commits):
     for commit in commits:
         for diff_obj in commit.diff(base_commit):
             # We always add new python files
-            if diff_obj.change_type in ["A", "M", "R"] and (
-                diff_obj.b_path.endswith(".py") or diff_obj.b_path.endswith(".mdx")
-            ):
+            if diff_obj.change_type in ["A"] and (diff_obj.b_path.endswith(".py") or diff_obj.b_path.endswith(".mdx")):
                 code_diff.append(diff_obj.b_path)
+            # Now for modified files
+            elif (
+                diff_obj.change_type in ["M", "R"]
+                and diff_obj.b_path.endswith(".py")
+                or diff_obj.b_path.endswith(".mdx")
+            ):
+                # In case of renames, we'll look at the tests using both the old and new name.
+                if diff_obj.a_path != diff_obj.b_path:
+                    code_diff.extend([diff_obj.a_path, diff_obj.b_path])
+                else:
+                    # Otherwise, we check modifications are in code and not docstrings.
+                    if diff_is_docstring_only(repo, commit, diff_obj.b_path):
+                        print(f"Ignoring diff in {diff_obj.b_path} as it only concerns docstrings or comments.")
+                    else:
+                        code_diff.append(diff_obj.a_path)
 
     return code_diff