[CI] Better reports #2 (#8163)

.github/workflows/self-scheduled.yml

@@ -63,7 +63,12 @@ jobs:
           source .env/bin/activate
           python -m pytest -n 1 --dist=loadfile -s --make_reports=tests tests
 
+      - name: Failure short reports
+        if: ${{ always() }}
+        run: cat reports/report_tests_failures_short.txt
+        
       - name: Run examples tests on GPU
+        if: ${{ always() }}
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
@@ -73,7 +78,12 @@ jobs:
           pip install -r examples/requirements.txt
           python -m pytest -n 1 --dist=loadfile -s --make_reports=examples examples
 
+      - name: Failure short reports
+        if: ${{ always() }}
+        run: cat reports/report_examples_failures_short.txt
+
       - name: Run all pipeline tests on GPU
+        if: ${{ always() }}
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
@@ -83,11 +93,15 @@ jobs:
           source .env/bin/activate
           python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_pipeline tests
 
-      - name: test suite reports artifacts
+      - name: Failure short reports
+        if: ${{ always() }}
+        run: cat reports/report_tests_pipeline_failures_short.txt
+
+      - name: Test suite reports artifacts
         if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
-          name: test_reports
+          name: run_all_tests_torch_and_tf_gpu_test_reports
           path: reports
 
 

src/transformers/testing_utils.py

@@ -725,18 +725,22 @@ def pytest_terminal_summary_main(tr, id):
     orig_tbstyle = config.option.tbstyle
     orig_reportchars = tr.reportchars
 
-    report_files = dict(
-        durations="durations",
-        short_summary="short_summary",
-        summary_errors="errors",
-        summary_failures="failures",
-        summary_warnings="warnings",
-        summary_passes="passes",
-        summary_stats="stats",
-    )
     dir = "reports"
     Path(dir).mkdir(parents=True, exist_ok=True)
-    report_files.update((k, f"{dir}/report_{id}_{v}.txt") for k, v in report_files.items())
+    report_files = {
+        k: f"{dir}/report_{id}_{k}.txt"
+        for k in [
+            "durations",
+            "errors",
+            "failures_long",
+            "failures_short",
+            "failures_line",
+            "passes",
+            "stats",
+            "summary_short",
+            "warnings",
+        ]
+    }
 
     # custom durations report
     # note: there is no need to call pytest --durations=XX to get this separate report
@@ -757,34 +761,60 @@ def pytest_terminal_summary_main(tr, id):
                     break
                 f.write(f"{rep.duration:02.2f}s {rep.when:<8} {rep.nodeid}\n")
 
+    def summary_failures_short(tr):
+        # expecting that the reports were --tb=long (default) so we chop them off here to the last frame
+        reports = tr.getreports("failed")
+        if not reports:
+            return
+        tr.write_sep("=", "FAILURES SHORT STACK")
+        for rep in reports:
+            msg = tr._getfailureheadline(rep)
+            tr.write_sep("_", msg, red=True, bold=True)
+            # chop off the optional leading extra frames, leaving only the last one
+            longrepr = re.sub(r".*_ _ _ (_ ){10,}_ _ ", "", rep.longreprtext, 0, re.M | re.S)
+            tr._tw.line(longrepr)
+            # note: not printing out any rep.sections to keep the report short
+
     # use ready-made report funcs, we are just hijacking the filehandle to log to a dedicated file each
     # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/terminal.py#L814
     # note: some pytest plugins may interfere by hijacking the default `terminalreporter` (e.g.
     # pytest-instafail does that)
-    tr.reportchars = "wPpsxXEf"  # emulate -rA (used in summary_passes() and short_test_summary())
+
-    config.option.tbstyle = "auto"
+    # report failures with line/short/long styles
-    with open(report_files["summary_failures"], "w") as f:
+    config.option.tbstyle = "auto"  # full tb
+    with open(report_files["failures_long"], "w") as f:
         tr._tw = create_terminal_writer(config, f)
         tr.summary_failures()
 
-    with open(report_files["summary_errors"], "w") as f:
+    # config.option.tbstyle = "short" # short tb
+    with open(report_files["failures_short"], "w") as f:
+        tr._tw = create_terminal_writer(config, f)
+        summary_failures_short(tr)
+
+    config.option.tbstyle = "line"  # one line per error
+    with open(report_files["failures_line"], "w") as f:
+        tr._tw = create_terminal_writer(config, f)
+        tr.summary_failures()
+
+    with open(report_files["errors"], "w") as f:
         tr._tw = create_terminal_writer(config, f)
         tr.summary_errors()
 
-    with open(report_files["summary_warnings"], "w") as f:
+    with open(report_files["warnings"], "w") as f:
         tr._tw = create_terminal_writer(config, f)
         tr.summary_warnings()  # normal warnings
         tr.summary_warnings()  # final warnings
 
-    with open(report_files["summary_passes"], "w") as f:
+    tr.reportchars = "wPpsxXEf"  # emulate -rA (used in summary_passes() and short_test_summary())
+    with open(report_files["passes"], "w") as f:
         tr._tw = create_terminal_writer(config, f)
         tr.summary_passes()
 
-    with open(report_files["short_summary"], "w") as f:
+    with open(report_files["summary_short"], "w") as f:
         tr._tw = create_terminal_writer(config, f)
         tr.short_test_summary()
 
-    with open(report_files["summary_stats"], "w") as f:
+    with open(report_files["stats"], "w") as f:
         tr._tw = create_terminal_writer(config, f)
         tr.summary_stats()