diff --git a/utils/notification_service.py b/utils/notification_service.py index 650d76b28c..610d597d23 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -314,7 +314,7 @@ class Message: return entries_changed @property - def model_failures(self) -> Dict: + def model_failures(self) -> List[Dict]: # Obtain per-model failures def per_model_sum(model_category_dict): return dicts_to_sum(model_category_dict["failed"].values()) diff --git a/utils/notification_service_doc_tests.py b/utils/notification_service_doc_tests.py index aac768fb39..c516963be1 100644 --- a/utils/notification_service_doc_tests.py +++ b/utils/notification_service_doc_tests.py @@ -19,7 +19,7 @@ import os import re import time from fnmatch import fnmatch -from typing import Dict +from typing import Dict, List import requests from slack_sdk import WebClient @@ -132,30 +132,42 @@ class Message: } @property - def category_failures(self) -> Dict: + def category_failures(self) -> List[Dict]: + failure_blocks = [] + + MAX_ERROR_TEXT = 3000 - len("The following examples had failures:\n\n\n\n") - len("[Truncated]\n") line_length = 40 category_failures = {k: v["failed"] for k, v in doc_test_results.items() if isinstance(v, dict)} - report = "" - for category, failures in category_failures.items(): + def single_category_failures(category, failures): + text = "" if len(failures) == 0: + return "" + text += f"*{category} failures*:".ljust(line_length // 2).rjust(line_length // 2) + "\n" + + for idx, failure in enumerate(failures): + new_text = text + f"`{failure}`\n" + if len(new_text) > MAX_ERROR_TEXT: + text = text + "[Truncated]\n" + break + text = new_text + + return text + + for category, failures in category_failures.items(): + report = single_category_failures(category, failures) + if len(report) == 0: continue + block = { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"The following examples had failures:\n\n\n{report}\n", + }, + } + failure_blocks.append(block) - if report != "": - report += "\n\n" - - report += f"*{category} failures*:".ljust(line_length // 2).rjust(line_length // 2) + "\n" - report += "`" - report += "`\n`".join(failures) - report += "`" - - return { - "type": "section", - "text": { - "type": "mrkdwn", - "text": f"The following examples had failures:\n\n\n{report}\n", - }, - } + return failure_blocks @property def payload(self) -> str: @@ -165,7 +177,7 @@ class Message: blocks.append(self.failures) if self.n_failures > 0: - blocks.extend([self.category_failures]) + blocks.extend(self.category_failures) if self.n_failures == 0: blocks.append(self.no_failures) @@ -211,10 +223,19 @@ class Message: ) def get_reply_blocks(self, job_name, job_link, failures, text): - failures_text = "" + # `text` must be less than 3001 characters in Slack SDK + # keep some room for adding "[Truncated]" when necessary + MAX_ERROR_TEXT = 3000 - len("[Truncated]") + + failure_text = "" for key, value in failures.items(): - value = value[:200] + " [Truncated]" if len(value) > 250 else value - failures_text += f"*{key}*\n_{value}_\n\n" + new_text = failure_text + f"*{key}*\n_{value}_\n\n" + if len(new_text) > MAX_ERROR_TEXT: + # `failure_text` here has length <= 3000 + failure_text = failure_text + "[Truncated]" + break + # `failure_text` here has length <= MAX_ERROR_TEXT + failure_text = new_text title = job_name content = {"type": "section", "text": {"type": "mrkdwn", "text": text}} @@ -229,7 +250,7 @@ class Message: return [ {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}}, content, - {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}}, + {"type": "section", "text": {"type": "mrkdwn", "text": failure_text}}, ] def post_reply(self):