Doc checks (#25408)

* Document check_dummies

* Type hints and doc in other files

* Document check inits

* Add documentation to

* Address review comments

utils/check_dummies.py

@@ -12,10 +12,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""
+This script is responsible for making sure the dummies in utils/dummies_xxx.py are up to date with the main init.
 
+Why dummies? This is to make sure that a user can always import all objects from `transformers`, even if they don't
+have the necessary extra libs installed. Those objects will then raise helpful error message whenever the user tries
+to access one of their methods.
+
+Usage (from the root of the repo):
+
+Check that the dummy files are up to date (used in `make repo-consistency`):
+
+```bash
+python utils/check_dummies.py
+```
+
+Update the dummy files if needed (used in `make fix-copies`):
+
+```bash
+python utils/check_dummies.py --fix_and_overwrite
+```
+"""
 import argparse
 import os
 import re
+from typing import Dict, List, Optional
 
 
 # All paths are set with the intent you should run this script from the root of the repo with the command
@@ -26,13 +47,16 @@ PATH_TO_TRANSFORMERS = "src/transformers"
 _re_backend = re.compile(r"is\_([a-z_]*)_available()")
 # Matches from xxx import bla
 _re_single_line_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
+# Matches if not is_xxx_available()
 _re_test_backend = re.compile(r"^\s+if\s+not\s+\(?is\_[a-z_]*\_available\(\)")
 
 
+# Template for the dummy objects.
 DUMMY_CONSTANT = """
 {0} = None
 """
 
+
 DUMMY_CLASS = """
 class {0}(metaclass=DummyObject):
     _backends = {1}
@@ -48,8 +72,18 @@ def {0}(*args, **kwargs):
 """
 
 
-def find_backend(line):
-    """Find one (or multiple) backend in a code line of the init."""
+def find_backend(line: str) -> Optional[str]:
+    """
+    Find one (or multiple) backend in a code line of the init.
+
+    Args:
+        line (`str`): A code line in an init file.
+
+    Returns:
+        Optional[`str`]: If one (or several) backend is found, returns it. In the case of multiple backends (the line
+        contains `if is_xxx_available() and `is_yyy_available()`) returns all backends joined on `_and_` (so
+        `xxx_and_yyy` for instance).
+    """
     if _re_test_backend.search(line) is None:
         return None
     backends = [b[0] for b in _re_backend.findall(line)]
@@ -57,8 +91,13 @@ def find_backend(line):
     return "_and_".join(backends)
 
 
-def read_init():
-    """Read the init and extracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects."""
+def read_init() -> Dict[str, List[str]]:
+    """
+    Read the init and extract backend-specific objects.
+
+    Returns:
+        Dict[str, List[str]]: A dictionary mapping backend name to the list of object names requiring that backend.
+    """
     with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r", encoding="utf-8", newline="\n") as f:
         lines = f.readlines()
 
@@ -83,8 +122,10 @@ def read_init():
                 line = lines[line_index]
                 single_line_import_search = _re_single_line_import.search(line)
                 if single_line_import_search is not None:
+                    # Single-line imports
                     objects.extend(single_line_import_search.groups()[0].split(", "))
                 elif line.startswith(" " * 12):
+                    # Multiple-line imports (with 3 indent level)
                     objects.append(line[12:-2])
                 line_index += 1
 
@@ -95,8 +136,17 @@ def read_init():
     return backend_specific_objects
 
 
-def create_dummy_object(name, backend_name):
-    """Create the code for the dummy object corresponding to `name`."""
+def create_dummy_object(name: str, backend_name: str) -> str:
+    """
+    Create the code for a dummy object.
+
+    Args:
+        name (`str`): The name of the object.
+        backend_name (`str`): The name of the backend required for that object.
+
+    Returns:
+        `str`: The code of the dummy object.
+    """
     if name.isupper():
         return DUMMY_CONSTANT.format(name)
     elif name.islower():
@@ -105,11 +155,21 @@ def create_dummy_object(name, backend_name):
         return DUMMY_CLASS.format(name, backend_name)
 
 
-def create_dummy_files(backend_specific_objects=None):
-    """Create the content of the dummy files."""
+def create_dummy_files(backend_specific_objects: Optional[Dict[str, List[str]]] = None) -> Dict[str, str]:
+    """
+    Create the content of the dummy files.
+
+    Args:
+        backend_specific_objects (`Dict[str, List[str]]`, *optional*):
+            The mapping backend name to list of backend-specific objects. If not passed, will be obtained by calling
+            `read_init()`.
+
+    Returns:
+        `Dict[str, str]`: A dictionary mapping backend name to code of the corresponding backend file.
+    """
     if backend_specific_objects is None:
         backend_specific_objects = read_init()
-    # For special correspondence backend to module name as used in the function requires_modulename
+
     dummy_files = {}
 
     for backend, objects in backend_specific_objects.items():
@@ -122,10 +182,17 @@ def create_dummy_files(backend_specific_objects=None):
     return dummy_files
 
 
-def check_dummies(overwrite=False):
-    """Check if the dummy files are up to date and maybe `overwrite` with the right content."""
+def check_dummies(overwrite: bool = False):
+    """
+    Check if the dummy files are up to date and maybe `overwrite` with the right content.
+
+    Args:
+        overwrite (`bool`, *optional*, default to `False`):
+            Whether or not to overwrite the content of the dummy files. Will raise an error if they are not up to date
+            when `overwrite=False`.
+    """
     dummy_files = create_dummy_files()
-    # For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
+    # For special correspondence backend name to shortcut as used in utils/dummy_xxx_objects.py
     short_names = {"torch": "pt"}
 
     # Locate actual dummy modules and read their content.
@@ -143,6 +210,7 @@ def check_dummies(overwrite=False):
         else:
             actual_dummies[backend] = ""
 
+    # Compare actual with what they should be.
     for backend in dummy_files.keys():
         if dummy_files[backend] != actual_dummies[backend]:
             if overwrite: