Document toc check and doctest check scripts (#25319)

* Clean doc toc check and make doctest list better

* Add to Makefile

utils/check_doctest_list.py

@@ -12,28 +12,65 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""
+This script is responsible for cleaning the list of doctests by making sure the entries all exist and are in
+alphabetical order.
 
+Usage (from the root of the repo):
+
+Check that the doctest list is properly sorted and all files exist (used in `make repo-consistency`):
+
+```bash
+python utils/check_doctest_list.py
+```
+
+Auto-sort the doctest list if it is not properly sorted (used in `make fix-copies`):
+
+```bash
+python utils/check_doctest_list.py --fix_and_overwrite
+```
+"""
+import argparse
 import os
 
 
 # All paths are set with the intent you should run this script from the root of the repo with the command
 # python utils/check_doctest_list.py
 REPO_PATH = "."
+DOCTEST_FILE_PATHS = ["documentation_tests.txt", "slow_documentation_tests.txt"]
 
 
-if __name__ == "__main__":
-    doctest_file_path = os.path.join(REPO_PATH, "utils/documentation_tests.txt")
+def clean_doctest_list(doctest_file, overwrite=False):
     non_existent_paths = []
     all_paths = []
-    with open(doctest_file_path) as fp:
-        for line in fp:
+    with open(doctest_file, "r", encoding="utf-8") as f:
+        for line in f:
             line = line.strip()
             path = os.path.join(REPO_PATH, line)
             if not (os.path.isfile(path) or os.path.isdir(path)):
                 non_existent_paths.append(line)
-            all_paths.append(path)
+            all_paths.append(line)
+
     if len(non_existent_paths) > 0:
-        non_existent_paths = "\n".join(non_existent_paths)
+        non_existent_paths = "\n".join([f"- {f}" for f in non_existent_paths])
         raise ValueError(f"`utils/documentation_tests.txt` contains non-existent paths:\n{non_existent_paths}")
-    if all_paths != sorted(all_paths):
-        raise ValueError("Files in `utils/documentation_tests.txt` are not in alphabetical order.")
+
+    sorted_paths = sorted(all_paths)
+    if all_paths != sorted_paths:
+        if not overwrite:
+            raise ValueError(
+                f"Files in `{doctest_file}` are not in alphabetical order, run `make fix-copies` to fix "
+                "this automatically."
+            )
+        with open(doctest_file, "w", encoding="utf-8") as f:
+            f.write("\n".join(sorted_paths) + "\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
+    args = parser.parse_args()
+
+    for doctest_file in DOCTEST_FILE_PATHS:
+        doctest_file = os.path.join(REPO_PATH, "utils", doctest_file)
+        clean_doctest_list(doctest_file, args.fix_and_overwrite)