SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

new failure CI reports for all jobs (#38298)

Browse Source 
* new failures

* report_repo_id

* report_repo_id

* report_repo_id

* More fixes

* More fixes

* More fixes

* ruff

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2025-05-24 19:15:02 +02:00
committed by GitHub
parent 31f8a0fe8a
commit d0c9c66d1c
11 changed files with 248 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
utils/notification_service.py
View File

@@ -30,8 +30,17 @@ from huggingface_hub import HfApi
from slack_sdk import WebClient
api = HfApi()
client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"])
# A map associating the job names (specified by `inputs.job` in a workflow file) with the keys of
# `additional_files`. This is used to remove some entries in `additional_files` that are not concerned by a
# specific job. See below.
job_to_test_map = {
"run_models_gpu": "Models",
"run_trainer_and_fsdp_gpu": "Trainer & FSDP",
"run_pipelines_torch_gpu": "PyTorch pipelines",
"run_pipelines_tf_gpu": "TensorFlow pipelines",
"run_examples_gpu": "Examples directory",
"run_torch_cuda_extensions_gpu": "DeepSpeed",
}
NON_MODEL_TEST_MODULES = [
"deepspeed",
@@ -516,6 +525,7 @@ class Message:
if len(self.selected_warnings) > 0:
blocks.append(self.warnings)
new_failure_blocks = []
for idx, (prev_workflow_run_id, prev_ci_artifacts) in enumerate(
[self.prev_ci_artifacts] + self.other_ci_artifacts
):
@@ -524,13 +534,11 @@ class Message:
new_failure_blocks = self.get_new_model_failure_blocks(
prev_ci_artifacts=prev_ci_artifacts, with_header=False
)
if len(new_failure_blocks) > 0:
blocks.extend(new_failure_blocks)
# To save the list of new model failures and uploaed to hub repositories
extra_blocks = self.get_new_model_failure_blocks(prev_ci_artifacts=prev_ci_artifacts, to_truncate=False)
if extra_blocks:
filename = "new_model_failures"
filename = "new_failures"
if idx > 0:
filename = f"{filename}_against_{prev_workflow_run_id}"
@@ -541,17 +549,17 @@ class Message:
# upload results to Hub dataset
file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.txt")
commit_info = api.upload_file(
_ = api.upload_file(
path_or_fileobj=file_path,
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/{filename}.txt",
repo_id="hf-internal-testing/transformers_daily_ci",
repo_id=report_repo_id,
repo_type="dataset",
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
)
url = f"https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/{commit_info.oid}/{report_repo_folder}/ci_results_{job_name}/{filename}.txt"
# extra processing to save to json format
new_failed_tests = {}
nb_new_failed_tests = 0
for line in failure_text.split():
if "https://github.com/huggingface/transformers/actions/runs" in line:
pattern = r"<(https://github.com/huggingface/transformers/actions/runs/.+?/job/.+?)\|(.+?)>"
@@ -563,36 +571,56 @@ class Message:
model = line.split("/")[1]
if model not in new_failed_tests:
new_failed_tests[model] = {"single-gpu": [], "multi-gpu": []}
for url, device in items:
for _, device in items:
new_failed_tests[model][f"{device}-gpu"].append(line)
nb_new_failed_tests += 1
file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.json")
with open(file_path, "w", encoding="UTF-8") as fp:
json.dump(new_failed_tests, fp, ensure_ascii=False, indent=4)
# upload results to Hub dataset
file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.json")
_ = api.upload_file(
commit_info = api.upload_file(
path_or_fileobj=file_path,
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/{filename}.json",
repo_id="hf-internal-testing/transformers_daily_ci",
repo_id=report_repo_id,
repo_type="dataset",
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
)
new_failures_url = f"https://huggingface.co/datasets/{report_repo_id}/raw/{commit_info.oid}/{report_repo_folder}/ci_results_{job_name}/{filename}.json"
if idx == 0:
block = {
"type": "section",
"text": {
"type": "plain_text",
"text": " ",
"type": "mrkdwn",
"text": f"*There are {nb_new_failed_tests} new failed tests*\n\n(compared to previous run: <https://github.com/huggingface/transformers/actions/runs/{prev_workflow_run_id}|{prev_workflow_run_id}>)",
},
"accessory": {
"type": "button",
"text": {"type": "plain_text", "text": "Check New model failures"},
"url": url,
"text": {"type": "plain_text", "text": "Check new failures"},
"url": new_failures_url,
},
}
blocks.append(block)
else:
block = {
"type": "section",
"text": {
"type": "mrkdwn",
# TODO: We should NOT assume it's always Nvidia CI, but it's the case at this moment.
"text": f"*There are {nb_new_failed_tests} failed tests unique to this run*\n\n(compared to Nvidia CI: <https://github.com/huggingface/transformers/actions/runs/{prev_workflow_run_id}|{prev_workflow_run_id}>)",
},
"accessory": {
"type": "button",
"text": {"type": "plain_text", "text": "Check failures"},
"url": new_failures_url,
},
}
blocks.append(block)
if len(new_failure_blocks) > 0:
blocks.extend(new_failure_blocks)
return json.dumps(blocks)
@@ -717,14 +745,28 @@ class Message:
if prev_ci_artifacts is None:
return []
sorted_dict = sorted(self.model_results.items(), key=lambda t: t[0])
if len(self.model_results) > 0:
target_results = self.model_results
else:
target_results = self.additional_results[job_to_test_map[job_name]]
# Make the format uniform between `model_results` and `additional_results[XXX]`
if "failures" in target_results:
target_results = {job_name: target_results}
sorted_dict = sorted(target_results.items(), key=lambda t: t[0])
job = job_to_test_map[job_name]
prev_model_results = {}
if (
f"ci_results_{job_name}" in prev_ci_artifacts
and "model_results.json" in prev_ci_artifacts[f"ci_results_{job_name}"]
and f"{test_to_result_name[job]}_results.json" in prev_ci_artifacts[f"ci_results_{job_name}"]
):
prev_model_results = json.loads(prev_ci_artifacts[f"ci_results_{job_name}"]["model_results.json"])
prev_model_results = json.loads(
prev_ci_artifacts[f"ci_results_{job_name}"][f"{test_to_result_name[job]}_results.json"]
)
# Make the format uniform between `model_results` and `additional_results[XXX]`
if "failures" in prev_model_results:
prev_model_results = {job_name: prev_model_results}
all_failure_lines = {}
for job, job_result in sorted_dict:
@@ -751,7 +793,7 @@ class Message:
all_failure_lines[new_text].append(f"<{url}|{device}>" if url is not None else device)
MAX_ERROR_TEXT = 3000 - len("[Truncated]") - len("```New model failures```\n\n")
MAX_ERROR_TEXT = 3000 - len("[Truncated]") - len("```New failures```\n\n")
if not to_truncate:
MAX_ERROR_TEXT = float("inf")
failure_text = ""
@@ -768,10 +810,10 @@ class Message:
if failure_text:
if with_header:
blocks.append(
{"type": "header", "text": {"type": "plain_text", "text": "New model failures", "emoji": True}}
{"type": "header", "text": {"type": "plain_text", "text": "New failures", "emoji": True}}
)
else:
failure_text = f"*New model failures*\n\n{failure_text}"
failure_text = f"{failure_text}"
blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": failure_text}})
return blocks
@@ -927,6 +969,9 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
if __name__ == "__main__":
api = HfApi()
client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"])
SLACK_REPORT_CHANNEL_ID = os.environ["SLACK_REPORT_CHANNEL"]
# runner_status = os.environ.get("RUNNER_STATUS")
@@ -1157,15 +1202,7 @@ if __name__ == "__main__":
elif ci_event.startswith("Push CI (AMD)"):
additional_files = {}
# A map associating the job names (specified by `inputs.job` in a workflow file) with the keys of
# `additional_files`. This is used to remove some entries in `additional_files` that are not concerned by a
# specific job. See below.
job_to_test_map = {
"run_pipelines_torch_gpu": "PyTorch pipelines",
"run_pipelines_tf_gpu": "TensorFlow pipelines",
"run_examples_gpu": "Examples directory",
"run_torch_cuda_extensions_gpu": "DeepSpeed",
}
report_repo_id = os.getenv("REPORT_REPO_ID")
# if it is not a scheduled run, upload the reports to a subfolder under `report_repo_folder`
report_repo_subfolder = ""
@@ -1258,81 +1295,100 @@ if __name__ == "__main__":
os.makedirs(os.path.join(os.getcwd(), f"ci_results_{job_name}"))
nvidia_daily_ci_workflow = "huggingface/transformers/.github/workflows/self-scheduled-caller.yml"
amd_daily_ci_workflows = (
"huggingface/transformers/.github/workflows/self-scheduled-amd-mi210-caller.yml",
"huggingface/transformers/.github/workflows/self-scheduled-amd-mi250-caller.yml",
)
is_nvidia_daily_ci_workflow = os.environ.get("GITHUB_WORKFLOW_REF").startswith(nvidia_daily_ci_workflow)
is_amd_daily_ci_workflow = os.environ.get("GITHUB_WORKFLOW_REF").startswith(amd_daily_ci_workflows)
is_scheduled_ci_run = os.environ.get("GITHUB_EVENT_NAME") == "schedule"
# For AMD workflow runs: the different AMD CI callers (MI210/MI250/MI300, etc.) are triggered by `workflow_run`
# event of `.github/workflows/self-scheduled-amd-caller.yml`.
if is_amd_daily_ci_workflow:
# Get the path to the file on the runner that contains the full event webhook payload.
event_payload_path = os.environ.get("GITHUB_EVENT_PATH")
# Load the event payload
with open(event_payload_path) as fp:
event_payload = json.load(fp)
# The event that triggers the `workflow_run` event.
if "workflow_run" in event_payload:
is_scheduled_ci_run = event_payload["event"] == "schedule"
# Only the model testing job is concerned: this condition is to avoid other jobs to upload the empty list as
# results.
if job_name == "run_models_gpu":
with open(f"ci_results_{job_name}/model_results.json", "w", encoding="UTF-8") as fp:
json.dump(model_results, fp, indent=4, ensure_ascii=False)
api.upload_file(
path_or_fileobj=f"ci_results_{job_name}/model_results.json",
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/model_results.json",
repo_id="hf-internal-testing/transformers_daily_ci",
repo_type="dataset",
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
)
# Let's create a file contain job --> job link
model_job_links = {}
sorted_dict = sorted(model_results.items(), key=lambda t: t[0])
for job, job_result in sorted_dict:
model_name = job
if model_name.startswith("models_"):
model_name = model_name[len("models_") :]
model_job_links[model_name] = job_result["job_link"]
with open(f"ci_results_{job_name}/model_job_links.json", "w", encoding="UTF-8") as fp:
json.dump(model_job_links, fp, indent=4, ensure_ascii=False)
api.upload_file(
path_or_fileobj=f"ci_results_{job_name}/model_job_links.json",
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/model_job_links.json",
repo_id="hf-internal-testing/transformers_daily_ci",
repo_type="dataset",
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
)
# Must have the same keys as in `additional_results`.
# The values are used as the file names where to save the corresponding CI job results.
test_to_result_name = {
"Models": "model",
"Trainer & FSDP": "trainer_and_fsdp",
"PyTorch pipelines": "torch_pipeline",
"TensorFlow pipelines": "tf_pipeline",
"Examples directory": "example",
"DeepSpeed": "deepspeed",
}
for job, job_result in additional_results.items():
with open(f"ci_results_{job_name}/{test_to_result_name[job]}_results.json", "w", encoding="UTF-8") as fp:
json.dump(job_result, fp, indent=4, ensure_ascii=False)
test_name_and_result_pairs = []
if len(model_results) > 0:
test_name = job_to_test_map[job_name]
test_name_and_result_pairs.append((test_name, model_results))
for test_name, result in additional_results.items():
test_name_and_result_pairs.append((test_name, result))
for test_name, result in test_name_and_result_pairs:
with open(f"ci_results_{job_name}/{test_to_result_name[test_name]}_results.json", "w", encoding="UTF-8") as fp:
json.dump(result, fp, indent=4, ensure_ascii=False)
api.upload_file(
path_or_fileobj=f"ci_results_{job_name}/{test_to_result_name[job]}_results.json",
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/{test_to_result_name[job]}_results.json",
repo_id="hf-internal-testing/transformers_daily_ci",
path_or_fileobj=f"ci_results_{job_name}/{test_to_result_name[test_name]}_results.json",
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/{test_to_result_name[test_name]}_results.json",
repo_id=report_repo_id,
repo_type="dataset",
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
)
# Let's create a file contain job --> job link
if len(model_results) > 0:
target_results = model_results
else:
target_results = additional_results[job_to_test_map[job_name]]
# Make the format uniform between `model_results` and `additional_results[XXX]`
if "failures" in target_results:
target_results = {job_name: target_results}
job_links = {}
sorted_dict = sorted(target_results.items(), key=lambda t: t[0])
for job, job_result in sorted_dict:
if job.startswith("models_"):
job = job[len("models_") :]
job_links[job] = job_result["job_link"]
with open(f"ci_results_{job_name}/job_links.json", "w", encoding="UTF-8") as fp:
json.dump(job_links, fp, indent=4, ensure_ascii=False)
api.upload_file(
path_or_fileobj=f"ci_results_{job_name}/job_links.json",
path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/job_links.json",
repo_id=report_repo_id,
repo_type="dataset",
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
)
prev_workflow_run_id = None
other_workflow_run_ids = []
if is_scheduled_ci_run:
# TODO: remove `if job_name == "run_models_gpu"`
if job_name == "run_models_gpu":
prev_workflow_run_id = get_last_daily_ci_workflow_run_id(
token=os.environ["ACCESS_REPO_INFO_TOKEN"], workflow_id=workflow_id
prev_workflow_run_id = get_last_daily_ci_workflow_run_id(
token=os.environ["ACCESS_REPO_INFO_TOKEN"], workflow_id=workflow_id
)
# For a scheduled run that is not the Nvidia's scheduled daily CI, add Nvidia's scheduled daily CI run as a target to compare.
if not is_nvidia_daily_ci_workflow:
# The id of the workflow `.github/workflows/self-scheduled-caller.yml` (not of a workflow run of it).
other_workflow_id = "90575235"
# We need to get the Nvidia's scheduled daily CI run that match the current run (i.e. run with the same commit SHA)
other_workflow_run_id = get_last_daily_ci_workflow_run_id(
token=os.environ["ACCESS_REPO_INFO_TOKEN"], workflow_id=other_workflow_id, commit_sha=ci_sha
)
# For a scheduled run that is not the Nvidia's scheduled daily CI, add Nvidia's scheduled daily CI run as a target to compare.
if not is_nvidia_daily_ci_workflow:
# The id of the workflow `.github/workflows/self-scheduled-caller.yml` (not of a workflow run of it).
other_workflow_id = "90575235"
# We need to get the Nvidia's scheduled daily CI run that match the current run (i.e. run with the same commit SHA)
other_workflow_run_id = get_last_daily_ci_workflow_run_id(
token=os.environ["ACCESS_REPO_INFO_TOKEN"], workflow_id=other_workflow_id, commit_sha=ci_sha
)
other_workflow_run_ids.append(other_workflow_run_id)
other_workflow_run_ids.append(other_workflow_run_id)
else:
prev_workflow_run_id = os.environ["PREV_WORKFLOW_RUN_ID"]
other_workflow_run_id = os.environ["OTHER_WORKFLOW_RUN_ID"]
@@ -1359,13 +1415,6 @@ if __name__ == "__main__":
else:
other_ci_artifacts.append((target_workflow_run_id, ci_artifacts))
job_to_test_map.update(
{
"run_models_gpu": "Models",
"run_trainer_and_fsdp_gpu": "Trainer & FSDP",
}
)
ci_name_in_report = ""
if job_name in job_to_test_map:
ci_name_in_report = job_to_test_map[job_name]