From 83f025125d636ec9907535b986c9373541ba578e Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Tue, 13 Jul 2021 21:13:18 +0200 Subject: [PATCH] Add timeout to CI. (#12684) * Global 60-300 seconds timeout * Add verbose option * [skip ci] typo --- .github/workflows/self-push.yml | 9 ++++---- .github/workflows/self-scheduled.yml | 23 ++++++++++--------- setup.py | 4 ++-- src/transformers/dependency_versions_table.py | 2 +- utils/notification_service.py | 2 +- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index 6d05c79274..b6f51496b7 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -18,6 +18,7 @@ env: TRANSFORMERS_IS_CI: yes OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 + PYTEST_TIMEOUT: 60 jobs: run_tests_torch_gpu: @@ -48,7 +49,7 @@ jobs: - name: Run all non-slow tests on GPU run: | - python -m pytest -n 2 --dist=loadfile --make-reports=tests_torch_gpu tests + python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_gpu tests - name: Failure short reports if: ${{ always() }} @@ -134,7 +135,7 @@ jobs: env: MKL_SERVICE_FORCE_INTEL: 1 run: | - python -m pytest -n 2 --dist=loadfile --make-reports=tests_torch_multi_gpu tests + python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_multi_gpu tests - name: Failure short reports if: ${{ always() }} @@ -217,7 +218,7 @@ jobs: - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended + python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended - name: Failure short reports if: ${{ always() }} @@ -258,7 +259,7 @@ jobs: - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_torch_cuda_extensions_multi_gpu tests/deepspeed tests/extended + python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_multi_gpu tests/deepspeed tests/extended - name: Failure short reports if: ${{ always() }} diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index d6aff136c8..e01340bf18 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -14,6 +14,7 @@ env: RUN_SLOW: yes OMP_NUM_THREADS: 16 MKL_NUM_THREADS: 16 + PYTEST_TIMEOUT: 300 jobs: run_all_tests_torch_gpu: @@ -44,7 +45,7 @@ jobs: - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_torch_gpu tests + python -m pytest -n 1 -v --dist=loadfile --make-reports=tests_torch_gpu tests - name: Failure short reports if: ${{ always() }} @@ -60,7 +61,7 @@ jobs: TRANSFORMERS_IS_CI: yes run: | pip install -r examples/pytorch/_tests_requirements.txt - python -m pytest -n 1 --dist=loadfile --make-reports=examples_torch_gpu examples + python -m pytest -n 1 -v --dist=loadfile --make-reports=examples_torch_gpu examples - name: Failure short reports if: ${{ always() }} @@ -71,7 +72,7 @@ jobs: env: RUN_PIPELINE_TESTS: yes run: | - python -m pytest -n 1 --dist=loadfile -m is_pipeline_test --make-reports=tests_torch_pipeline_gpu tests + python -m pytest -n 1 -v --dist=loadfile -m is_pipeline_test --make-reports=tests_torch_pipeline_gpu tests - name: Failure short reports if: ${{ always() }} @@ -112,7 +113,7 @@ jobs: TF_NUM_INTEROP_THREADS: 1 TF_NUM_INTRAOP_THREADS: 16 run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_gpu tests + python -m pytest -n 1 -v --dist=loadfile --make-reports=tests_tf_gpu tests - name: Failure short reports if: ${{ always() }} @@ -125,7 +126,7 @@ jobs: TF_NUM_INTEROP_THREADS: 1 TF_NUM_INTRAOP_THREADS: 16 run: | - python -m pytest -n 1 --dist=loadfile -m is_pipeline_test --make-reports=tests_tf_pipeline_gpu tests + python -m pytest -n 1 -v --dist=loadfile -m is_pipeline_test --make-reports=tests_tf_pipeline_gpu tests - name: Failure short reports if: ${{ always() }} @@ -168,7 +169,7 @@ jobs: env: MKL_SERVICE_FORCE_INTEL: 1 run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_torch_multi_gpu tests + python -m pytest -n 1 -v --dist=loadfile --make-reports=tests_torch_multi_gpu tests - name: Failure short reports if: ${{ always() }} @@ -179,7 +180,7 @@ jobs: env: RUN_PIPELINE_TESTS: yes run: | - python -m pytest -n 1 --dist=loadfile -m is_pipeline_test --make-reports=tests_torch_pipeline_multi_gpu tests + python -m pytest -n 1 -v --dist=loadfile -m is_pipeline_test --make-reports=tests_torch_pipeline_multi_gpu tests - name: Failure short reports if: ${{ always() }} @@ -220,7 +221,7 @@ jobs: TF_NUM_INTEROP_THREADS: 1 TF_NUM_INTRAOP_THREADS: 16 run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_multi_gpu tests + python -m pytest -n 1 -v --dist=loadfile --make-reports=tests_tf_multi_gpu tests - name: Failure short reports if: ${{ always() }} @@ -233,7 +234,7 @@ jobs: TF_NUM_INTEROP_THREADS: 1 TF_NUM_INTRAOP_THREADS: 16 run: | - python -m pytest -n 1 --dist=loadfile -m is_pipeline_test --make-reports=tests_tf_pipeline_multi_gpu tests + python -m pytest -n 1 -v --dist=loadfile -m is_pipeline_test --make-reports=tests_tf_pipeline_multi_gpu tests - name: Failure short reports if: ${{ always() }} @@ -274,7 +275,7 @@ jobs: - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended + python -m pytest -n 1 -v --dist=loadfile --make-reports=tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended - name: Failure short reports if: ${{ always() }} @@ -315,7 +316,7 @@ jobs: - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile --make-reports=tests_torch_cuda_extensions_multi_gpu tests/deepspeed tests/extended + python -m pytest -n 1 -v --dist=loadfile --make-reports=tests_torch_cuda_extensions_multi_gpu tests/deepspeed tests/extended - name: Failure short reports if: ${{ always() }} diff --git a/setup.py b/setup.py index 953e651aff..56f46b4885 100644 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ _deps = [ "pyyaml>=5.1", "pydantic", "pytest", - "pytest-sugar", + "pytest-timeout", "pytest-xdist", "python>=3.6.0", "ray[tune]", @@ -259,7 +259,7 @@ extras["codecarbon"] = deps_list("codecarbon") extras["sentencepiece"] = deps_list("sentencepiece", "protobuf") extras["testing"] = ( deps_list( - "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-sugar", "black", "sacrebleu", "rouge-score", "nltk" + "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-timeout", "black", "sacrebleu", "rouge-score", "nltk" ) + extras["retrieval"] + extras["modelcreation"] diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index f078623674..681b985f53 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -39,7 +39,7 @@ deps = { "pyyaml": "pyyaml>=5.1", "pydantic": "pydantic", "pytest": "pytest", - "pytest-sugar": "pytest-sugar", + "pytest-timeout": "pytest-timeout", "pytest-xdist": "pytest-xdist", "python": "python>=3.6.0", "ray[tune]": "ray[tune]", diff --git a/utils/notification_service.py b/utils/notification_service.py index 900b77ba9d..94d7b1f230 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -44,7 +44,7 @@ def format_for_slack(total_results, results, scheduled: bool): "type": "header", "text": { "type": "plain_text", - "text": "🤗 Results of the scheduled tests, March 11, 2021." if scheduled else "🤗 Self-push results", + "text": "🤗 Results of the scheduled tests." if scheduled else "🤗 Self-push results", "emoji": True, }, }