diff --git a/utils/notification_service.py b/utils/notification_service.py index e751794539..7838338ba2 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -75,6 +75,8 @@ def handle_test_results(test_results): failed = 0 success = 0 + errors = 0 + skipped = 0 # When the output is short enough, the output is surrounded by = signs: "== OUTPUT ==" # When it is too long, those signs are not present. @@ -83,10 +85,14 @@ def handle_test_results(test_results): for i, expression in enumerate(expressions): if "failed" in expression: failed += int(expressions[i - 1]) + if "errors" in expression: + errors += int(expressions[i - 1]) if "passed" in expression: success += int(expressions[i - 1]) + if "skipped" in expression: + skipped += int(expressions[i - 1]) - return failed, success, time_spent + return failed, errors, success, skipped, time_spent def handle_stacktraces(test_results): @@ -1188,7 +1194,9 @@ if __name__ == "__main__": matrix_job_results = { matrix_name: { "failed": {m: {"unclassified": 0, "single": 0, "multi": 0} for m in test_categories}, + "errors": 0, "success": 0, + "skipped": 0, "time_spent": "", "failures": {}, "job_link": {}, @@ -1213,8 +1221,10 @@ if __name__ == "__main__": # Link to the GitHub Action job job = artifact_name_to_job_map[path] matrix_job_results[matrix_name]["job_link"][artifact_gpu] = job["html_url"] - failed, success, time_spent = handle_test_results(artifact["stats"]) + failed, errors, success, skipped, time_spent = handle_test_results(artifact["stats"]) matrix_job_results[matrix_name]["success"] += success + matrix_job_results[matrix_name]["errors"] += errors + matrix_job_results[matrix_name]["skipped"] += skipped matrix_job_results[matrix_name]["time_spent"] += time_spent[1:-1] + ", " stacktraces = handle_stacktraces(artifact["failures_line"]) @@ -1317,7 +1327,9 @@ if __name__ == "__main__": additional_results = { key: { "failed": {"unclassified": 0, "single": 0, "multi": 0}, + "errors": 0, "success": 0, + "skipped": 0, "time_spent": "", "error": False, "failures": {}, @@ -1343,9 +1355,11 @@ if __name__ == "__main__": artifact = retrieve_artifact(path, artifact_gpu) stacktraces = handle_stacktraces(artifact["failures_line"]) - failed, success, time_spent = handle_test_results(artifact["stats"]) + failed, errors, success, skipped, time_spent = handle_test_results(artifact["stats"]) additional_results[key]["failed"][artifact_gpu or "unclassified"] += failed additional_results[key]["success"] += success + additional_results[key]["errors"] += errors + additional_results[key]["skipped"] += skipped additional_results[key]["time_spent"] += time_spent[1:-1] + ", " if len(artifact["errors"]):