From 9e9b8f1d99fa898bbcb5f270e1bedcd9012cd7fc Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 10 Aug 2021 14:54:52 +0200 Subject: [PATCH] Roll out the test fetcher on push tests (#13055) * Use test fetcher for push tests as well * Force diff with last commit for circleCI on master * Fix syntax error * Style * Schedule nightly tests --- .circleci/config.yml | 297 +++++++++++++++++++++++++++++++- .github/workflows/self-push.yml | 84 ++++++++- utils/tests_fetcher.py | 70 +++++--- 3 files changed, 421 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec29b0fb74..1b15a00881 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,6 +97,37 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_torch_and_tf_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.6 + environment: + OMP_NUM_THREADS: 1 + RUN_PT_TF_CROSS_TESTS: yes + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-torch_and_tf-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: pip install --upgrade pip + - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,speech,vision] + - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html + - save_cache: + key: v0.4-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf tests -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_torch_and_flax: working_directory: ~/transformers @@ -133,6 +164,37 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_torch_and_flax_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.6 + environment: + OMP_NUM_THREADS: 1 + RUN_PT_FLAX_CROSS_TESTS: yes + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-torch_and_flax-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: pip install --upgrade pip + - run: pip install .[sklearn,flax,torch,testing,sentencepiece,speech,vision] + - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html + - save_cache: + key: v0.4-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax tests -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_torch: working_directory: ~/transformers @@ -168,6 +230,36 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_torch_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + OMP_NUM_THREADS: 1 + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-torch-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: pip install --upgrade pip + - run: pip install .[sklearn,torch,testing,sentencepiece,speech,vision,timm] + - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html + - save_cache: + key: v0.4-torch-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 3 --dist=loadfile -s --make-reports=tests_torch tests | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_tf: working_directory: ~/transformers @@ -201,6 +293,34 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_tf_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + OMP_NUM_THREADS: 1 + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-tf-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: pip install --upgrade pip + - run: pip install .[sklearn,tf-cpu,testing,sentencepiece] + - save_cache: + key: v0.4-tf-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf tests | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_flax: working_directory: ~/transformers @@ -234,6 +354,34 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_flax_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + OMP_NUM_THREADS: 1 + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-flax-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: pip install --upgrade pip + - run: sudo pip install .[flax,testing,sentencepiece] + - save_cache: + key: v0.4-flax-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax tests | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_pipelines_torch: working_directory: ~/transformers @@ -270,6 +418,37 @@ jobs: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_pipelines_torch_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + OMP_NUM_THREADS: 1 + RUN_PIPELINE_TESTS: yes + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-torch-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: pip install --upgrade pip + - run: pip install .[sklearn,torch,testing,sentencepiece,speech,vision] + - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html + - save_cache: + key: v0.4-torch-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test tests | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_pipelines_tf: working_directory: ~/transformers @@ -305,6 +484,35 @@ jobs: - store_artifacts: path: ~/transformers/reports + run_tests_pipelines_tf_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + OMP_NUM_THREADS: 1 + RUN_PIPELINE_TESTS: yes + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-tf-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: pip install --upgrade pip + - run: pip install .[sklearn,tf-cpu,testing,sentencepiece] + - save_cache: + key: v0.4-tf-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf $(cat test_list.txt) -m tests | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports + run_tests_custom_tokenizers: working_directory: ~/transformers docker: @@ -399,8 +607,45 @@ jobs: path: ~/transformers/test_preparation.txt - run: | if [ -f test_list.txt ]; then - python -m pytest -sv $(cat test_list.txt) -m is_staging_test + python -m pytest -sv --make-reports=tests_hub $(cat test_list.txt) -m is_staging_test | tee tests_output.txt fi + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports + + run_tests_hub_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + HUGGINGFACE_CO_STAGING: yes + RUN_GIT_LFS_TESTS: yes + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-hub-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: sudo apt-get install git-lfs + - run: | + git config --global user.email "ci@dummy.com" + git config --global user.name "ci" + - run: pip install --upgrade pip + - run: pip install .[torch,sentencepiece,testing] + - save_cache: + key: v0.4-hub-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -sv --make-reports=tests_hub tests -m is_staging_test | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports run_tests_onnxruntime: working_directory: ~/transformers @@ -428,12 +673,41 @@ jobs: path: ~/transformers/test_preparation.txt - run: | if [ -f test_list.txt ]; then - python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch $(cat test_list.txt) -k onnx | tee tests_output.txt + python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_onnx $(cat test_list.txt) -k onnx | tee tests_output.txt fi - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: path: ~/transformers/reports + + run_tests_onnxruntime_all: + working_directory: ~/transformers + docker: + - image: circleci/python:3.7 + environment: + OMP_NUM_THREADS: 1 + TRANSFORMERS_IS_CI: yes + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - restore_cache: + keys: + - v0.4-torch-{{ checksum "setup.py" }} + - v0.4-{{ checksum "setup.py" }} + - run: pip install --upgrade pip + - run: pip install .[torch,testing,sentencepiece,onnxruntime] + - save_cache: + key: v0.4-onnx-{{ checksum "setup.py" }} + paths: + - '~/.cache/pip' + - run: | + python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_onnx tests -k onnx | tee tests_output.txt + - store_artifacts: + path: ~/transformers/tests_output.txt + - store_artifacts: + path: ~/transformers/reports + build_doc: working_directory: ~/transformers docker: @@ -579,6 +853,25 @@ workflows: - run_tests_hub - build_doc - deploy_doc: *workflow_filters + nightly: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + jobs: + - run_tests_torch_and_tf_all + - run_tests_torch_and_flax_all + - run_tests_torch_all + - run_tests_tf_all + - run_tests_flax_all + - run_tests_pipelines_torch_all + - run_tests_pipelines_tf_all + - run_tests_onnxruntime_all + - run_tests_hub_all + # tpu_testing_jobs: # triggers: # - schedule: diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index b6f51496b7..32f36a5cc2 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -46,10 +46,22 @@ jobs: python -c "import torch; print('Cuda version:', torch.version.cuda)" python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + + - name: Fetch the tests to run + run: | + python utils/tests_fetcher.py | tee test_preparation.txt + + - name: Report fetched tests + uses: actions/upload-artifact@v2 + with: + name: test_fetched + path: test_preparation.txt - name: Run all non-slow tests on GPU run: | - python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_gpu tests + if [ -f test_list.txt ]; then + python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_gpu $(cat test_list.txt) + fi - name: Failure short reports if: ${{ always() }} @@ -86,12 +98,24 @@ jobs: # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" # +# - name: Fetch the tests to run +# run: | +# python utils/tests_fetcher.py | tee test_preparation.txt +# +# - name: Report fetched tests +# uses: actions/upload-artifact@v2 +# with: +# name: test_fetched +# path: test_preparation.txt +# # - name: Run all non-slow tests on GPU # env: # TF_NUM_INTRAOP_THREADS: 8 # TF_NUM_INTEROP_THREADS: 1 # run: | -# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_gpu tests +# if [ -f test_list.txt ]; then +# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_gpu $(cat test_list.txt) +# fi # # - name: Failure short reports # if: ${{ always() }} @@ -131,11 +155,23 @@ jobs: python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + - name: Fetch the tests to run + run: | + python utils/tests_fetcher.py | tee test_preparation.txt + + - name: Report fetched tests + uses: actions/upload-artifact@v2 + with: + name: test_fetched + path: test_preparation.txt + - name: Run all non-slow tests on GPU env: MKL_SERVICE_FORCE_INTEL: 1 run: | - python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_multi_gpu tests + if [ -f test_list.txt ]; then + python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_multi_gpu $(cat test_list.txt) + fi - name: Failure short reports if: ${{ always() }} @@ -172,12 +208,24 @@ jobs: # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" # +# - name: Fetch the tests to run +# run: | +# python utils/tests_fetcher.py | tee test_preparation.txt +# +# - name: Report fetched tests +# uses: actions/upload-artifact@v2 +# with: +# name: test_fetched +# path: test_preparation.txt +# # - name: Run all non-slow tests on GPU # env: # TF_NUM_INTRAOP_THREADS: 8 # TF_NUM_INTEROP_THREADS: 1 # run: | -# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_multi_gpu tests +# if [ -f test_list.txt ]; then +# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_multi_gpu $(cat test_list.txt) +# fi # # - name: Failure short reports # if: ${{ always() }} @@ -215,10 +263,22 @@ jobs: python -c "import torch; print('Cuda version:', torch.version.cuda)" python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + + - name: Fetch the tests to run + run: | + python utils/tests_fetcher.py --filters tests/deepspeed tests/extended | tee test_preparation.txt + + - name: Report fetched tests + uses: actions/upload-artifact@v2 + with: + name: test_fetched + path: test_preparation.txt - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended + if [ -f test_list.txt ]; then + python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_gpu $(cat test_list.txt) + fi - name: Failure short reports if: ${{ always() }} @@ -257,9 +317,21 @@ jobs: python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" + - name: Fetch the tests to run + run: | + python utils/tests_fetcher.py --filters tests/deepspeed tests/extended | tee test_preparation.txt + + - name: Report fetched tests + uses: actions/upload-artifact@v2 + with: + name: test_fetched + path: test_preparation.txt + - name: Run all tests on GPU run: | - python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_multi_gpu tests/deepspeed tests/extended + if [ -f test_list.txt ]; then + python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_multi_gpu $(cat test_list.txt) + fi - name: Failure short reports if: ${{ always() }} diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 6377e58894..c6d5fcf32b 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -82,23 +82,39 @@ def diff_is_docstring_only(repo, branching_point, filename): return old_content_clean == new_content_clean -def get_modified_python_files(): +def get_modified_python_files(diff_with_last_commit=False): """ - Return a list of python files that have been modified between the current head and the master branch. + Return a list of python files that have been modified between: + + - the current head and the master branch if `diff_with_last_commit=False` (default) + - the current head and its parent commit otherwise. """ repo = Repo(PATH_TO_TRANFORMERS) - print(f"Master is at {repo.refs.master.commit}") - print(f"Current head is at {repo.head.commit}") + if not diff_with_last_commit: + print(f"Master is at {repo.refs.master.commit}") + print(f"Current head is at {repo.head.commit}") - branching_commits = repo.merge_base(repo.refs.master, repo.head) - for commit in branching_commits: - print(f"Branching commit: {commit}") + branching_commits = repo.merge_base(repo.refs.master, repo.head) + for commit in branching_commits: + print(f"Branching commit: {commit}") + return get_diff(repo, repo.head.commit, branching_commits) + else: + print(f"Master is at {repo.head.commit}") + parent_commits = repo.head.commit.parents + for commit in parent_commits: + print(f"Parent commit: {commit}") + return get_diff(repo, repo.head.commit, parent_commits) + +def get_diff(repo, base_commit, commits): + """ + Get's the diff between one or several commits and the head of the repository. + """ print("\n### DIFF ###\n") code_diff = [] - for commit in branching_commits: - for diff_obj in commit.diff(repo.head.commit): + for commit in commits: + for diff_obj in commit.diff(base_commit): # We always add new python files if diff_obj.change_type == "A" and diff_obj.b_path.endswith(".py"): code_diff.append(diff_obj.b_path) @@ -365,8 +381,8 @@ def sanity_check(): ) -def infer_tests_to_run(output_file): - modified_files = get_modified_python_files() +def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None): + modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}") # Create the map that will give us all impacted modules. @@ -396,6 +412,10 @@ def infer_tests_to_run(output_file): # Remove duplicates test_files_to_run = sorted(list(set(test_files_to_run))) + if filters is not None: + for filter in filters: + test_files_to_run = [f for f in test_files_to_run if f.startswith(filter)] + print(f"\n### TEST TO RUN ###\n{_print_list(test_files_to_run)}") if len(test_files_to_run) > 0: with open(output_file, "w", encoding="utf-8") as f: @@ -410,22 +430,28 @@ if __name__ == "__main__": parser.add_argument( "--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run" ) + parser.add_argument( + "--diff_with_last_commit", + action="store_true", + help="To fetch the tests between the current commit and the last commit", + ) + parser.add_argument( + "--filters", type=str, nargs="*", help="Only keep the test files matching one of those filters." + ) args = parser.parse_args() if args.sanity_check: sanity_check() else: repo = Repo(PATH_TO_TRANFORMERS) - # For now we run all tests on the master branch. After testing this more and making sure it works most of the - # time, we will apply the same logic to the tests on the master branch and only run the whole suite once per - # day. + + diff_with_last_commit = args.diff_with_last_commit if not repo.head.is_detached and repo.head.ref == repo.refs.master: - print("Master branch detected, running all tests.") + print("Master branch detected, fetching tests against last commit.") + diff_with_last_commit = True + + try: + infer_tests_to_run(args.output_file, diff_with_last_commit=diff_with_last_commit, filters=args.filters) + except Exception as e: + print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.") with open(args.output_file, "w", encoding="utf-8") as f: f.write("./tests/") - else: - try: - infer_tests_to_run(args.output_file) - except Exception as e: - print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.") - with open(args.output_file, "w", encoding="utf-8") as f: - f.write("./tests/")