SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

(Re-)Enable Nightly + Past CI (#22393)

Browse Source 
* Enable Nightly + Past CI

* put schedule

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2023-03-30 21:06:35 +02:00
committed by GitHub
parent d5de578c22
commit 0fe6c6bdca
13 changed files with 446 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
utils/get_ci_error_statistics.py
View File

@@ -66,12 +66,12 @@ def get_artifacts_links(worflow_run_id, token=None):
def download_artifact(artifact_name, artifact_url, output_dir, token):
"""Download a GitHub Action artifact from a URL.
The URL is of the from `https://api.github.com/repos/huggingface/transformers/actions/artifacts/{ARTIFACT_ID}/zip`,
The URL is of the form `https://api.github.com/repos/huggingface/transformers/actions/artifacts/{ARTIFACT_ID}/zip`,
but it can't be used to download directly. We need to get a redirect URL first.
See https://docs.github.com/en/rest/actions/artifacts#download-an-artifact
"""
# Get the redirect URL first
cmd = f'curl -v -H "Accept: application/vnd.github+json" -H "Authorization: token {token}" {artifact_url}'
cmd = f'curl -v -H "Accept: application/vnd.github+json" -H "Authorization: Bearer {token}" {artifact_url}'
output = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
o = output.stdout.decode("utf-8")
lines = o.splitlines()

32
utils/notification_service.py
View File

@@ -590,23 +590,20 @@ class Message:
time.sleep(1)
def retrieve_artifact(name: str, gpu: Optional[str]):
def retrieve_artifact(artifact_path: str, gpu: Optional[str]):
if gpu not in [None, "single", "multi"]:
raise ValueError(f"Invalid GPU for artifact. Passed GPU: `{gpu}`.")
if gpu is not None:
name = f"{gpu}-gpu_{name}"
_artifact = {}
if os.path.exists(name):
files = os.listdir(name)
if os.path.exists(artifact_path):
files = os.listdir(artifact_path)
for file in files:
try:
with open(os.path.join(name, file)) as f:
with open(os.path.join(artifact_path, file)) as f:
_artifact[file.split(".")[0]] = f.read()
except UnicodeDecodeError as e:
raise ValueError(f"Could not open {os.path.join(name, file)}.") from e
raise ValueError(f"Could not open {os.path.join(artifact_path, file)}.") from e
return _artifact
@@ -629,8 +626,14 @@ def retrieve_available_artifacts():
directories = filter(os.path.isdir, os.listdir())
for directory in directories:
if directory.startswith("single-gpu"):
artifact_name = directory[len("single-gpu") + 1 :]
artifact_name = directory
name_parts = artifact_name.split("_postfix_")
if len(name_parts) > 1:
artifact_name = name_parts[0]
if artifact_name.startswith("single-gpu"):
artifact_name = artifact_name[len("single-gpu") + 1 :]
if artifact_name in _available_artifacts:
_available_artifacts[artifact_name].single_gpu = True
@@ -639,7 +642,7 @@ def retrieve_available_artifacts():
_available_artifacts[artifact_name].add_path(directory, gpu="single")
elif directory.startswith("multi-gpu"):
elif artifact_name.startswith("multi-gpu"):
artifact_name = directory[len("multi-gpu") + 1 :]
if artifact_name in _available_artifacts:
@@ -649,7 +652,6 @@ def retrieve_available_artifacts():
_available_artifacts[artifact_name].add_path(directory, gpu="multi")
else:
artifact_name = directory
if artifact_name not in _available_artifacts:
_available_artifacts[artifact_name] = Artifact(artifact_name)
@@ -805,10 +807,12 @@ if __name__ == "__main__":
framework, version = ci_event.replace("Past CI - ", "").split("-")
framework = "PyTorch" if framework == "pytorch" else "TensorFlow"
job_name_prefix = f"{framework} {version}"
elif ci_event.startswith("Nightly CI"):
job_name_prefix = "Nightly CI"
for model in model_results.keys():
for artifact_path in available_artifacts[f"run_all_tests_gpu_{model}_test_reports"].paths:
artifact = retrieve_artifact(artifact_path["name"], artifact_path["gpu"])
artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"])
if "stats" in artifact:
# Link to the GitHub Action job
# The job names use `matrix.folder` which contain things like `models/bert` instead of `models_bert`
@@ -901,7 +905,7 @@ if __name__ == "__main__":
else:
additional_results[key]["job_link"][artifact_path["gpu"]] = github_actions_job_links.get(key)
artifact = retrieve_artifact(artifact_path["name"], artifact_path["gpu"])
artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"])
stacktraces = handle_stacktraces(artifact["failures_line"])
failed, success, time_spent = handle_test_results(artifact["stats"])

98
utils/past_ci_versions.py
View File

@@ -4,6 +4,18 @@ import os
past_versions_testing = {
"pytorch": {
"1.13": {
"torch": "1.13.1",
"torchvision": "0.14.1",
"torchaudio": "0.13.1",
"python": 3.9,
"cuda": "cu116",
"install": (
"python3 -m pip install --no-cache-dir -U torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1"
" --extra-index-url https://download.pytorch.org/whl/cu116"
),
"base_image": "nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04",
},
"1.12": {
"torch": "1.12.1",
"torchvision": "0.13.1",
@@ -14,6 +26,7 @@ past_versions_testing = {
"python3 -m pip install --no-cache-dir -U torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1"
" --extra-index-url https://download.pytorch.org/whl/cu113"
),
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"1.11": {
"torch": "1.11.0",
@@ -25,6 +38,7 @@ past_versions_testing = {
"python3 -m pip install --no-cache-dir -U torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0"
" --extra-index-url https://download.pytorch.org/whl/cu113"
),
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"1.10": {
"torch": "1.10.2",
@@ -36,6 +50,7 @@ past_versions_testing = {
"python3 -m pip install --no-cache-dir -U torch==1.10.2 torchvision==0.11.3 torchaudio==0.10.2"
" --extra-index-url https://download.pytorch.org/whl/cu113"
),
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
# torchaudio < 0.10 has no CUDA-enabled binary distributions
"1.9": {
@@ -48,87 +63,44 @@ past_versions_testing = {
"python3 -m pip install --no-cache-dir -U torch==1.9.1 torchvision==0.10.1 torchaudio==0.9.1"
" --extra-index-url https://download.pytorch.org/whl/cu111"
),
},
"1.8": {
"torch": "1.8.1",
"torchvision": "0.9.1",
"torchaudio": "0.8.1",
"python": 3.9,
"cuda": "cu111",
"install": (
"python3 -m pip install --no-cache-dir -U torch==1.8.1 torchvision==0.9.1 torchaudio==0.8.1"
" --extra-index-url https://download.pytorch.org/whl/cu111"
),
},
"1.7": {
"torch": "1.7.1",
"torchvision": "0.8.2",
"torchaudio": "0.7.2",
"python": 3.9,
"cuda": "cu110",
"install": (
"python3 -m pip install --no-cache-dir -U torch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2"
" --extra-index-url https://download.pytorch.org/whl/cu110"
),
},
"1.6": {
"torch": "1.6.0",
"torchvision": "0.7.0",
"torchaudio": "0.6.0",
"python": 3.8,
"cuda": "cu101",
"install": (
"python3 -m pip install --no-cache-dir -U torch==1.6.0 torchvision==0.7.0 torchaudio==0.6.0"
" --extra-index-url https://download.pytorch.org/whl/cu101"
),
},
"1.5": {
"torch": "1.5.1",
"torchvision": "0.6.1",
"torchaudio": "0.5.1",
"python": 3.8,
"cuda": "cu101",
"install": (
"python3 -m pip install --no-cache-dir -U torch==1.5.1 torchvision==0.6.1 torchaudio==0.5.1"
" --extra-index-url https://download.pytorch.org/whl/cu101"
),
},
"1.4": {
"torch": "1.4.0",
"torchvision": "0.5.0",
"torchaudio": "0.4.0",
"python": 3.8,
"cuda": "cu100",
"install": (
"python3 -m pip install --no-cache-dir -U torch==1.4.0 torchvision==0.5.0 torchaudio==0.4.0"
" --extra-index-url https://download.pytorch.org/whl/cu100"
),
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
},
"tensorflow": {
"2.11": {
"tensorflow": "2.11.1",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.11.1",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"2.10": {
"tensorflow": "2.10.1",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.10.1",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"2.9": {
"tensorflow": "2.9.3",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.9.3",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"2.8": {
"tensorflow": "2.8.2",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.8.2",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"2.7": {
"tensorflow": "2.7.3",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.7.3",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"2.6": {
"tensorflow": "2.6.5",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.6.5",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
"2.5": {
"tensorflow": "2.5.3",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.5.3",
},
# need another `nvidia:cuda` docker image, otherwise GPU not working
"2.4": {
"tensorflow": "2.4.4",
"install": "python3 -m pip install --no-cache-dir -U tensorflow==2.4.4",
# This should be specified as a docker build argument.
# We keep the information here for reference only.
"base_docker": "nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04",
"base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04",
},
},
}