General PR slow CI (#30540)

* More general PR slow CI

* Update utils/pr_slow_ci_models.py

Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com>

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com>

.github/workflows/self-pr-slow-ci.yml

@@ -4,6 +4,11 @@ on:
   pull_request:
     paths:
       - "src/transformers/models/*/modeling_*.py"
+      - "tests/models/*/test_*.py"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 
 env:
   HF_HOME: /mnt/cache
@@ -20,31 +25,46 @@ env:
   CUDA_VISIBLE_DEVICES: 0,1
 
 jobs:
-  check_for_new_model:
+  find_models_to_run:
       runs-on: ubuntu-22.04
-      name: Check if a PR is a new model PR
+      name: Find models to run slow tests
+      # Triggered only if the required label `run-slow` is added
+      if: ${{ contains(github.event.pull_request.labels.*.name, 'run-slow') }}
       outputs:
-        new_model: ${{ steps.check_new_model.outputs.new_model }}
+        models: ${{ steps.models_to_run.outputs.models }}
       steps:
         - uses: actions/checkout@v4
           with:
             fetch-depth: "0"
+            ref: ${{ github.event.pull_request.head.sha }}
 
-        - name: Check if there is a new model
-          id: check_new_model
+        - name: Get commit message
           run: |
+            echo "commit_message=$(git show -s --format=%s)" >> $GITHUB_ENV
+
+        - name: Get models to run slow tests
+          run: |
+            echo "${{ env.commit_message }}"
             python -m pip install GitPython
-            echo "new_model=$(python utils/check_if_new_model_added.py | tail -n 1)" >> $GITHUB_OUTPUT
+            python utils/pr_slow_ci_models.py --commit_message "${{ env.commit_message }}" | tee output.txt
+            echo "models=$(tail -n 1 output.txt)" >> $GITHUB_ENV
+
+        - name: Models to run slow tests
+          id: models_to_run
+          run: |
+            echo "${{ env.models }}"
+            echo "models=${{ env.models }}" >> $GITHUB_OUTPUT
 
   run_models_gpu:
-      name: Run all tests for the new model
-      # Triggered if it is a new model PR and the required label is added
-      if: ${{ needs.check_for_new_model.outputs.new_model != '' && contains(github.event.pull_request.labels.*.name, 'single-model-run-slow') }}
-      needs: check_for_new_model
+      name: Run all tests for the model
+      # Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run
+      # (either a new model PR or via a commit message)
+      if: ${{ needs.find_models_to_run.outputs.models != '[]' }}
+      needs: find_models_to_run
       strategy:
         fail-fast: false
         matrix:
-          folders: ["${{ needs.check_for_new_model.outputs.new_model }}"]
+          folders: ${{ fromJson(needs.find_models_to_run.outputs.models) }}
           machine_type: [single-gpu, multi-gpu]
       runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, ci]
       container:

utils/pr_slow_ci_models.py

@@ -13,15 +13,20 @@
 # limitations under the License.
 
 """
-This script is used to get the directory of the modeling file that is added in a pull request (i.e. a new model PR).
+This script is used to get the models for which to run slow CI.
+
+A new model added in a pull request will be included, as well as models specified in a commit message with a prefix
+`[run-slow]`, `[run_slow]` or `[run slow]`. For example, the commit message `[run_slow]bert, gpt2` will give `bert` and
+`gpt2`.
 
 Usage:
 
 ```bash
-python utils/check_if_new_model_added.py
+python utils/pr_slow_ci_models.py.py
 ```
 """
 
+import argparse
 import re
 from pathlib import Path
 from typing import List
@@ -82,7 +87,7 @@ def get_new_python_files() -> List[str]:
     return get_new_python_files_between_commits(repo.head.commit, branching_commits)
 
 
-if __name__ == "__main__":
+def get_new_model():
     new_files = get_new_python_files()
     reg = re.compile(r"src/transformers/(models/.*)/modeling_.*\.py")
 
@@ -93,4 +98,48 @@ if __name__ == "__main__":
             new_model = find_new_model[0]
             # It's unlikely we have 2 new modeling files in a pull request.
             break
-    print(new_model)
+    return new_model
+
+
+def parse_commit_message(commit_message: str) -> str:
+    """
+    Parses the commit message to find the models specified in it to run slow CI.
+
+    Args:
+        commit_message (`str`): The commit message of the current commit.
+
+    Returns:
+        `str`: The substring in `commit_message` after `[run-slow]`, [run_slow]` or [run slow]`. If no such prefix is
+         found, the empty string is returned.
+    """
+    if commit_message is None:
+        return ""
+
+    command_search = re.search(r"\[([^\]]*)\](.*)", commit_message)
+    if command_search is None:
+        return ""
+
+    command = command_search.groups()[0]
+    command = command.lower().replace("-", " ").replace("_", " ")
+    run_slow = command == "run slow"
+    if run_slow:
+        models = command_search.groups()[1].strip()
+        return models
+    else:
+        return ""
+
+
+def get_models(commit_message: str):
+    models = parse_commit_message(commit_message)
+    return [f"models/{x}" for x in models.replace(",", " ").split()]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--commit_message", type=str, default="", help="The commit message.")
+    args = parser.parse_args()
+
+    new_model = get_new_model()
+    specified_models = get_models(args.commit_message)
+    models = ([] if new_model == "" else [new_model]) + specified_models
+    print(models)