[CI] generate separate report files as artifacts (#7995)

* better reports

* a whole bunch of reports in their own files

* clean up

* improvements

* github artifacts experiment

* style

* complete the report generator with multiple improvements/fixes

* fix

* save all reports under one dir to easy upload

* can remove temp failing tests

* doc fix

* some cleanup

tests/conftest.py

@@ -11,7 +11,6 @@ from os.path import abspath, dirname, join
 git_repo_path = abspath(join(dirname(dirname(__file__)), "src"))
 sys.path.insert(1, git_repo_path)
 
-
 # silence FutureWarning warnings in tests since often we can't act on them until
 # they become normal warnings - i.e. the tests still need to test the current functionality
 warnings.simplefilter(action="ignore", category=FutureWarning)
@@ -22,3 +21,20 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers", "is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested"
     )
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--make_reports",
+        action="store",
+        default=False,
+        help="generate report files - the value will be used as a `report_`+val+`reportname.txt`",
+    )
+
+
+def pytest_terminal_summary(terminalreporter):
+    from transformers.testing_utils import pytest_terminal_summary_main
+
+    make_reports = terminalreporter.config.getoption("--make_reports")
+    if make_reports:
+        pytest_terminal_summary_main(terminalreporter, id=make_reports)