Tests: detect lines removed from "utils/not_doctested.txt" and doctest ALL generation files (#25763)

utils/not_doctested.txt

@@ -14,14 +14,12 @@ docs/source/en/custom_models.md
 docs/source/en/custom_tools.md
 docs/source/en/debugging.md
 docs/source/en/fast_tokenizers.md
-docs/source/en/generation_strategies.md
 docs/source/en/glossary.md
 docs/source/en/hpo_train.md
 docs/source/en/index.md
 docs/source/en/installation.md
 docs/source/en/internal/audio_utils.md
 docs/source/en/internal/file_utils.md
-docs/source/en/internal/generation_utils.md
 docs/source/en/internal/image_processing_utils.md
 docs/source/en/internal/modeling_utils.md
 docs/source/en/internal/pipelines_utils.md
@@ -45,7 +43,6 @@ docs/source/en/main_classes/output.md
 docs/source/en/main_classes/pipelines.md
 docs/source/en/main_classes/processors.md
 docs/source/en/main_classes/quantization.md
-docs/source/en/main_classes/text_generation.md
 docs/source/en/main_classes/tokenizer.md
 docs/source/en/main_classes/trainer.md
 docs/source/en/model_doc/albert.md
@@ -367,16 +364,6 @@ src/transformers/dynamic_module_utils.py
 src/transformers/feature_extraction_sequence_utils.py
 src/transformers/feature_extraction_utils.py
 src/transformers/file_utils.py
-src/transformers/generation/beam_constraints.py
-src/transformers/generation/beam_search.py
-src/transformers/generation/flax_logits_process.py
-src/transformers/generation/flax_utils.py
-src/transformers/generation/stopping_criteria.py
-src/transformers/generation/streamers.py
-src/transformers/generation/tf_logits_process.py
-src/transformers/generation_flax_utils.py
-src/transformers/generation_tf_utils.py
-src/transformers/generation_utils.py
 src/transformers/hf_argparser.py
 src/transformers/hyperparameter_search.py
 src/transformers/image_processing_utils.py
@@ -413,7 +400,6 @@ src/transformers/models/auto/modeling_tf_auto.py
 src/transformers/models/autoformer/configuration_autoformer.py
 src/transformers/models/autoformer/modeling_autoformer.py
 src/transformers/models/bark/convert_suno_to_hf.py
-src/transformers/models/bark/generation_configuration_bark.py
 src/transformers/models/bart/convert_bart_original_pytorch_checkpoint_to_pytorch.py
 src/transformers/models/bart/modeling_flax_bart.py
 src/transformers/models/bart/modeling_tf_bart.py
@@ -925,9 +911,7 @@ src/transformers/pipelines/object_detection.py
 src/transformers/pipelines/pt_utils.py
 src/transformers/pipelines/question_answering.py
 src/transformers/pipelines/table_question_answering.py
-src/transformers/pipelines/text2text_generation.py
 src/transformers/pipelines/text_classification.py
-src/transformers/pipelines/text_generation.py
 src/transformers/pipelines/token_classification.py
 src/transformers/pipelines/video_classification.py
 src/transformers/pipelines/visual_question_answering.py

utils/slow_documentation_tests.txt

@@ -1 +1,2 @@
+docs/source/en/generation_strategies.md
 docs/source/en/task_summary.md

utils/tests_fetcher.py

@@ -395,6 +395,31 @@ def get_all_doctest_files() -> List[str]:
     return sorted(test_files_to_run)
 
 
+def get_new_doctest_files(repo, base_commit, branching_commit) -> List[str]:
+    """
+    Get the list of files that were removed from "utils/not_doctested.txt", between `base_commit` and
+    `branching_commit`.
+
+    Returns:
+        `List[str]`: List of files that were removed from "utils/not_doctested.txt".
+    """
+    for diff_obj in branching_commit.diff(base_commit):
+        # Ignores all but the "utils/not_doctested.txt" file.
+        if diff_obj.a_path != "utils/not_doctested.txt":
+            continue
+        # Loads the two versions
+        folder = Path(repo.working_dir)
+        with checkout_commit(repo, branching_commit):
+            with open(folder / "utils/not_doctested.txt", "r", encoding="utf-8") as f:
+                old_content = f.read()
+        with open(folder / "utils/not_doctested.txt", "r", encoding="utf-8") as f:
+            new_content = f.read()
+        # Compute the removed lines and return them
+        removed_content = set(old_content.split("\n")) - set(new_content.split("\n"))
+        return sorted(removed_content)
+    return []
+
+
 def get_doctest_files(diff_with_last_commit: bool = False) -> List[str]:
     """
     Return a list of python and Markdown files where doc example have been modified between:
@@ -426,6 +451,10 @@ def get_doctest_files(diff_with_last_commit: bool = False) -> List[str]:
 
     all_test_files_to_run = get_all_doctest_files()
 
+    # Add to the test files to run any removed entry from "utils/not_doctested.txt".
+    new_test_files = get_new_doctest_files(repo, repo.head.commit, repo.refs.main.commit)
+    test_files_to_run = list(set(test_files_to_run + new_test_files))
+
     # Do not run slow doctest tests on CircleCI
     with open("utils/slow_documentation_tests.txt") as fp:
         slow_documentation_tests = set(fp.read().strip().split("\n"))