diff --git a/.github/workflows/check_runner_status.yml b/.github/workflows/check_runner_status.yml index 8912e32c94..7d0e3853b5 100644 --- a/.github/workflows/check_runner_status.yml +++ b/.github/workflows/check_runner_status.yml @@ -57,6 +57,7 @@ jobs: CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} + ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: runner status check RUNNER_STATUS: ${{ needs.check_runner_status.result }} OFFLINE_RUNNERS: ${{ needs.check_runner_status.outputs.offline_runners }} diff --git a/.github/workflows/self-nightly-scheduled.yml b/.github/workflows/self-nightly-scheduled.yml index accccf6164..ca5186e736 100644 --- a/.github/workflows/self-nightly-scheduled.yml +++ b/.github/workflows/self-nightly-scheduled.yml @@ -291,6 +291,7 @@ jobs: CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_PAST_FUTURE }} + ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: nightly-build RUNNER_STATUS: ${{ needs.check_runner_status.result }} RUNNER_ENV_STATUS: ${{ needs.check_runners.result }} diff --git a/.github/workflows/self-past.yml b/.github/workflows/self-past.yml index c59800445b..12ddcc6658 100644 --- a/.github/workflows/self-past.yml +++ b/.github/workflows/self-past.yml @@ -254,6 +254,7 @@ jobs: CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_PAST_FUTURE }} + ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: Past CI - ${{ inputs.framework }}-${{ inputs.version }} RUNNER_STATUS: ${{ needs.check_runner_status.result }} RUNNER_ENV_STATUS: ${{ needs.check_runners.result }} diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index b6c3a70e3e..be2fcc9880 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -568,6 +568,7 @@ jobs: CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }} + ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: push CI_TITLE_PUSH: ${{ github.event.head_commit.message }} CI_TITLE_WORKFLOW_RUN: ${{ github.event.workflow_run.head_commit.message }} diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 750f4a9569..f535efba27 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -482,6 +482,7 @@ jobs: CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} + ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: scheduled RUNNER_STATUS: ${{ needs.check_runner_status.result }} RUNNER_ENV_STATUS: ${{ needs.check_runners.result }} diff --git a/utils/extract_warnings.py b/utils/extract_warnings.py index bc26e79366..38c6ac5ecb 100644 --- a/utils/extract_warnings.py +++ b/utils/extract_warnings.py @@ -83,19 +83,14 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() # Required parameters - parser.add_argument( - "--workflow_run_id", default=None, type=str, required=True, help="A GitHub Actions workflow run id." - ) + parser.add_argument("--workflow_run_id", type=str, required=True, help="A GitHub Actions workflow run id.") parser.add_argument( "--output_dir", - default=None, type=str, required=True, help="Where to store the downloaded artifacts and other result files.", ) - parser.add_argument( - "--token", default=None, type=str, required=True, help="A token that has actions:read permission." - ) + parser.add_argument("--token", default=None, type=str, help="A token that has actions:read permission.") # optional parameters parser.add_argument( "--targets", @@ -119,7 +114,7 @@ if __name__ == "__main__": os.makedirs(args.output_dir, exist_ok=True) # get download links - artifacts = get_artifacts_links(args.workflow_run_id) + artifacts = get_artifacts_links(args.workflow_run_id, token=args.token) with open(os.path.join(args.output_dir, "artifacts.json"), "w", encoding="UTF-8") as fp: json.dump(artifacts, fp, ensure_ascii=False, indent=4) diff --git a/utils/get_ci_error_statistics.py b/utils/get_ci_error_statistics.py index 5e2846ee39..09dc4d7dd2 100644 --- a/utils/get_ci_error_statistics.py +++ b/utils/get_ci_error_statistics.py @@ -4,17 +4,22 @@ import math import os import subprocess import time +import traceback import zipfile from collections import Counter import requests -def get_job_links(workflow_run_id): +def get_job_links(workflow_run_id, token=None): """Extract job names and their job links in a GitHub Actions workflow run""" + headers = None + if token is not None: + headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"} + url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100" - result = requests.get(url).json() + result = requests.get(url, headers=headers).json() job_links = {} try: @@ -22,21 +27,25 @@ def get_job_links(workflow_run_id): pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100) for i in range(pages_to_iterate_over): - result = requests.get(url + f"&page={i + 2}").json() + result = requests.get(url + f"&page={i + 2}", headers=headers).json() job_links.update({job["name"]: job["html_url"] for job in result["jobs"]}) return job_links - except Exception as e: - print("Unknown error, could not fetch links.", e) + except Exception: + print(f"Unknown error, could not fetch links:\n{traceback.format_exc()}") return {} -def get_artifacts_links(worflow_run_id): +def get_artifacts_links(worflow_run_id, token=None): """Get all artifact links from a workflow run""" + headers = None + if token is not None: + headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"} + url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100" - result = requests.get(url).json() + result = requests.get(url, headers=headers).json() artifacts = {} try: @@ -44,12 +53,12 @@ def get_artifacts_links(worflow_run_id): pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100) for i in range(pages_to_iterate_over): - result = requests.get(url + f"&page={i + 2}").json() + result = requests.get(url + f"&page={i + 2}", headers=headers).json() artifacts.update({artifact["name"]: artifact["archive_download_url"] for artifact in result["artifacts"]}) return artifacts - except Exception as e: - print("Unknown error, could not fetch links.", e) + except Exception: + print(f"Unknown error, could not fetch links:\n{traceback.format_exc()}") return {} @@ -211,24 +220,19 @@ def make_github_table_per_model(reduced_by_model): if __name__ == "__main__": parser = argparse.ArgumentParser() # Required parameters - parser.add_argument( - "--workflow_run_id", default=None, type=str, required=True, help="A GitHub Actions workflow run id." - ) + parser.add_argument("--workflow_run_id", type=str, required=True, help="A GitHub Actions workflow run id.") parser.add_argument( "--output_dir", - default=None, type=str, required=True, help="Where to store the downloaded artifacts and other result files.", ) - parser.add_argument( - "--token", default=None, type=str, required=True, help="A token that has actions:read permission." - ) + parser.add_argument("--token", default=None, type=str, help="A token that has actions:read permission.") args = parser.parse_args() os.makedirs(args.output_dir, exist_ok=True) - _job_links = get_job_links(args.workflow_run_id) + _job_links = get_job_links(args.workflow_run_id, token=args.token) job_links = {} # To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee. # For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`. @@ -242,7 +246,7 @@ if __name__ == "__main__": with open(os.path.join(args.output_dir, "job_links.json"), "w", encoding="UTF-8") as fp: json.dump(job_links, fp, ensure_ascii=False, indent=4) - artifacts = get_artifacts_links(args.workflow_run_id) + artifacts = get_artifacts_links(args.workflow_run_id, token=args.token) with open(os.path.join(args.output_dir, "artifacts.json"), "w", encoding="UTF-8") as fp: json.dump(artifacts, fp, ensure_ascii=False, indent=4) diff --git a/utils/get_github_job_time.py b/utils/get_github_job_time.py index 5065c108aa..af59081ffd 100644 --- a/utils/get_github_job_time.py +++ b/utils/get_github_job_time.py @@ -1,5 +1,6 @@ import argparse import math +import traceback import dateutil.parser as date_parser import requests @@ -25,11 +26,15 @@ def extract_time_from_single_job(job): return job_info -def get_job_time(workflow_run_id): +def get_job_time(workflow_run_id, token=None): """Extract time info for all jobs in a GitHub Actions workflow run""" + headers = None + if token is not None: + headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"} + url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100" - result = requests.get(url).json() + result = requests.get(url, headers=headers).json() job_time = {} try: @@ -37,12 +42,12 @@ def get_job_time(workflow_run_id): pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100) for i in range(pages_to_iterate_over): - result = requests.get(url + f"&page={i + 2}").json() + result = requests.get(url + f"&page={i + 2}", headers=headers).json() job_time.update({job["name"]: extract_time_from_single_job(job) for job in result["jobs"]}) return job_time - except Exception as e: - print("Unknown error, could not fetch links.", e) + except Exception: + print(f"Unknown error, could not fetch links:\n{traceback.format_exc()}") return {} @@ -56,9 +61,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() # Required parameters - parser.add_argument( - "--workflow_run_id", default=None, type=str, required=True, help="A GitHub Actions workflow run id." - ) + parser.add_argument("--workflow_run_id", type=str, required=True, help="A GitHub Actions workflow run id.") args = parser.parse_args() job_time = get_job_time(args.workflow_run_id) diff --git a/utils/notification_service.py b/utils/notification_service.py index 9967db447e..a525a1eded 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -16,7 +16,6 @@ import ast import collections import functools import json -import math import operator import os import re @@ -25,6 +24,7 @@ import time from typing import Dict, List, Optional, Union import requests +from get_ci_error_statistics import get_job_links from slack_sdk import WebClient @@ -206,6 +206,15 @@ class Message: @property def warnings(self) -> Dict: + # If something goes wrong, let's avoid the CI report failing to be sent. + button_text = "Check warnings (Link not found)" + # Use the workflow run link + job_link = f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}" + if "Extract warnings in CI artifacts" in github_actions_job_links: + button_text = "Check warnings" + # Use the actual job link + job_link = f"{github_actions_job_links['Extract warnings in CI artifacts']}" + return { "type": "section", "text": { @@ -215,8 +224,8 @@ class Message: }, "accessory": { "type": "button", - "text": {"type": "plain_text", "text": "Check warnings", "emoji": True}, - "url": f"{github_actions_job_links['Extract warnings in CI artifacts']}", + "text": {"type": "plain_text", "text": button_text, "emoji": True}, + "url": job_link, }, } @@ -577,27 +586,6 @@ class Message: time.sleep(1) -def get_job_links(): - run_id = os.environ["GITHUB_RUN_ID"] - url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100" - result = requests.get(url).json() - jobs = {} - - try: - jobs.update({job["name"]: job["html_url"] for job in result["jobs"]}) - pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100) - - for i in range(pages_to_iterate_over): - result = requests.get(url + f"&page={i + 2}").json() - jobs.update({job["name"]: job["html_url"] for job in result["jobs"]}) - - return jobs - except Exception as e: - print("Unknown error, could not fetch links.", e) - - return {} - - def retrieve_artifact(name: str, gpu: Optional[str]): if gpu not in [None, "single", "multi"]: raise ValueError(f"Invalid GPU for artifact. Passed GPU: `{gpu}`.") @@ -770,7 +758,9 @@ if __name__ == "__main__": Message.error_out(title, ci_title) raise ValueError("Errored out.") - github_actions_job_links = get_job_links() + github_actions_job_links = get_job_links( + workflow_run_id=os.environ["GITHUB_RUN_ID"], token=os.environ["ACCESS_REPO_INFO_TOKEN"] + ) available_artifacts = retrieve_available_artifacts() modeling_categories = [ diff --git a/utils/past_ci_versions.py b/utils/past_ci_versions.py index a05e9d5f6c..c50bbb9b14 100644 --- a/utils/past_ci_versions.py +++ b/utils/past_ci_versions.py @@ -136,8 +136,10 @@ past_versions_testing = { if __name__ == "__main__": parser = argparse.ArgumentParser("Choose the framework and version to install") - parser.add_argument("--framework", help="The framework to install. Should be `torch` or `tensorflow`", type=str) - parser.add_argument("--version", help="The version of the framework to install.", type=str) + parser.add_argument( + "--framework", help="The framework to install. Should be `torch` or `tensorflow`", type=str, required=True + ) + parser.add_argument("--version", help="The version of the framework to install.", type=str, required=True) args = parser.parse_args() info = past_versions_testing[args.framework][args.version]