From 69b81c0a5ff4823750b8829497720d1648b633e9 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:31:24 -0400 Subject: [PATCH] Use a dynamic configuration for circleCI tests (#19325) * Generate config on the file * Fake modif for all test launch * Upload more artifacts * Typo and quality * Try converting th yml to txt * Leave my long lines alone yaml * Debug prints * Debug prints v2 * Try without sorting * Was it really working before? * Typo * Use a parameter * Use a parameter? * Typo * Here is some JSON * Another try * Learning to read... * Check default is used * Does this work? * With continuation * WiP * Use a parameter for test list * Other fake modif * With the comma * Name the test step so it doesn't blow up * Just one example modification * Final steps * Add nightlies * Move config generator * Add trigger for nightlies * Better workflow * Rebase on recent changes * Fix config creation * Fake modif in an example * Now fake modif in one config file * Fix install step in custom tokenizers test * Fix generated config * Better fix hopefully * Finally test modif in setup * final cleanup --- .circleci/config.yml | 810 ++-------------------------- .circleci/create_circleci_config.py | 391 ++++++++++++++ 2 files changed, 436 insertions(+), 765 deletions(-) create mode 100644 .circleci/create_circleci_config.py diff --git a/.circleci/config.yml b/.circleci/config.yml index f54a599808..d49a563e58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,65 +1,12 @@ version: 2.1 +setup: true orbs: - gcp-gke: circleci/gcp-gke@1.0.4 - go: circleci/go@1.3.0 - -# TPU REFERENCES -references: - checkout_ml_testing: &checkout_ml_testing - run: - name: Checkout ml-testing-accelerators - command: | - git clone https://github.com/GoogleCloudPlatform/ml-testing-accelerators.git - cd ml-testing-accelerators - git fetch origin 5e88ac24f631c27045e62f0e8d5dfcf34e425e25:stable - git checkout stable - build_push_docker: &build_push_docker - run: - name: Configure Docker - command: | - gcloud --quiet auth configure-docker - cd docker/transformers-pytorch-tpu - if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" . ; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi - docker push "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" - deploy_cluster: &deploy_cluster - run: - name: Deploy the job on the kubernetes cluster - command: | - go get github.com/google/go-jsonnet/cmd/jsonnet && \ - export PATH=$PATH:$HOME/go/bin && \ - kubectl create -f docker/transformers-pytorch-tpu/dataset.yaml || true && \ - job_name=$(jsonnet -J ml-testing-accelerators/ docker/transformers-pytorch-tpu/bert-base-cased.jsonnet --ext-str image=$GCR_IMAGE_PATH --ext-str image-tag=$CIRCLE_WORKFLOW_JOB_ID | kubectl create -f -) && \ - job_name=${job_name#job.batch/} && \ - job_name=${job_name% created} && \ - echo "Waiting on kubernetes job: $job_name" && \ - i=0 && \ - # 30 checks spaced 30s apart = 900s total. - max_checks=30 && \ - status_code=2 && \ - # Check on the job periodically. Set the status code depending on what - # happened to the job in Kubernetes. If we try max_checks times and - # still the job hasn't finished, give up and return the starting - # non-zero status code. - while [ $i -lt $max_checks ]; do ((i++)); if kubectl get jobs $job_name -o jsonpath='Failed:{.status.failed}' | grep "Failed:1"; then status_code=1 && break; elif kubectl get jobs $job_name -o jsonpath='Succeeded:{.status.succeeded}' | grep "Succeeded:1" ; then status_code=0 && break; else echo "Job not finished yet"; fi; sleep 30; done && \ - echo "Done waiting. Job status code: $status_code" && \ - pod_name=$(kubectl get po -l controller-uid=`kubectl get job $job_name -o "jsonpath={.metadata.labels.controller-uid}"` | awk 'match($0,!/NAME/) {print $1}') && \ - echo "GKE pod name: $pod_name" && \ - kubectl logs -f $pod_name --container=train - echo "Done with log retrieval attempt." && \ - gcloud container images delete "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" --force-delete-tags && \ - exit $status_code - delete_gke_jobs: &delete_gke_jobs - run: - name: Delete GKE Jobs - command: | - # Match jobs whose age matches patterns like '1h' or '1d', i.e. any job - # that has been around longer than 1hr. First print all columns for - # matches, then execute the delete. - kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $0}' - kubectl delete job $(kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $1}') - - + continuation: circleci/continuation@0.1.0 +parameters: + nightly: + type: boolean + default: false jobs: # Fetch the tests to run @@ -76,7 +23,7 @@ jobs: - run: mkdir -p test_preparation - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt - store_artifacts: - path: ~/transformers/tests_fetched_summary.txt + path: ~/transformers/tests_fetched_summary.txt - run: | if [ -f test_list.txt ]; then cp test_list.txt test_preparation/test_list.txt @@ -90,24 +37,30 @@ jobs: else touch test_preparation/filtered_test_list.txt fi + - run: python utils/tests_fetcher.py --filters tests examples | tee examples_tests_fetched_summary.txt + - run: | + if [ -f test_list.txt ]; then + mv test_list.txt test_preparation/examples_test_list.txt + else + touch test_preparation/examples_test_list.txt + fi + - store_artifacts: + path: test_preparation/test_list.txt - store_artifacts: path: ~/transformers/test_preparation/filtered_test_list.txt - - run: python utils/tests_fetcher.py --filters tests examples | tee examples_tests_fetched_summary.txt - store_artifacts: - path: ~/transformers/examples_tests_fetched_summary.txt + path: test_preparation/examples_test_list.txt + - run: python .circleci/create_circleci_config.py --fetcher_folder test_preparation - run: | - if [ -f test_list.txt ]; then - mv test_list.txt test_preparation/examples_test_list.txt - else - touch test_preparation/examples_test_list.txt - fi - - - persist_to_workspace: - root: test_preparation/ - paths: - test_list.txt - filtered_test_list.txt - examples_test_list.txt + if [ ! -s test_preparation/generated_config.yml ]; then + echo "No tests to run, exiting early!" + circleci-agent step halt + fi + - run: cp test_preparation/generated_config.yml test_preparation/generated_config.txt + - store_artifacts: + path: test_preparation/generated_config.txt + - continuation/continue: + configuration_path: test_preparation/generated_config.yml # To run all tests for the nightly build fetch_all_tests: @@ -116,516 +69,20 @@ jobs: - image: cimg/python:3.7.12 parallelism: 1 steps: + - run: pip install --upgrade pip + - run: pip install . - run: | mkdir test_preparation echo "tests" > test_preparation/test_list.txt echo "tests" > test_preparation/examples_test_list.txt - run: python utils/tests_fetcher.py --filter_pipeline_tests - run: mv test_list.txt test_preparation/filtered_test_list.txt - - - persist_to_workspace: - root: test_preparation/ - paths: - test_list.txt - - run_tests_torch_and_tf: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - RUN_PT_TF_CROSS_TESTS: yes - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-torch_and_tf-{{ checksum "setup.py" }} - - v0.5-torch_and_tf- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs - - run: git lfs install - - run: pip install --upgrade pip - - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision] - - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html - - run: pip install tensorflow_probability - - run: pip install https://github.com/kpu/kenlm/archive/master.zip - - run: pip install git+https://github.com/huggingface/accelerate - - save_cache: - key: v0.5-torch_and_tf-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf $(cat test_preparation/filtered_test_list.txt) -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt + - run: python .circleci/create_circleci_config.py --fetcher_folder test_preparation + - run: cp test_preparation/generated_config.yml test_preparation/generated_config.txt - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_torch_and_flax: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - RUN_PT_FLAX_CROSS_TESTS: yes - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-torch_and_flax-{{ checksum "setup.py" }} - - v0.5-torch_and_flax- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - - run: pip install --upgrade pip - - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision] - - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html - - run: pip install https://github.com/kpu/kenlm/archive/master.zip - - run: pip install git+https://github.com/huggingface/accelerate - - save_cache: - key: v0.5-torch_and_flax-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax $(cat test_preparation/filtered_test_list.txt) -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_torch: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-torch-{{ checksum "setup.py" }} - - v0.5-torch- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time - - run: pip install --upgrade pip - - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm] - - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html - - run: pip install https://github.com/kpu/kenlm/archive/master.zip - - run: pip install git+https://github.com/huggingface/accelerate - - save_cache: - key: v0.5-torch-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 3 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_torch $(cat test_preparation/filtered_test_list.txt) | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_tf: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-tf-{{ checksum "setup.py" }} - - v0.5-tf- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - - run: pip install --upgrade pip - - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision] - - run: pip install tensorflow_probability - - run: pip install https://github.com/kpu/kenlm/archive/master.zip - - save_cache: - key: v0.5-tf-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_tf $(cat test_preparation/filtered_test_list.txt) | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_flax: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-flax-{{ checksum "setup.py" }} - - v0.5-flax- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - - run: pip install --upgrade pip - - run: pip install .[flax,testing,sentencepiece,flax-speech,vision] - - run: pip install https://github.com/kpu/kenlm/archive/master.zip - - save_cache: - key: v0.5-flax-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_flax $(cat test_preparation/filtered_test_list.txt) | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_pipelines_torch: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-torch-{{ checksum "setup.py" }} - - v0.5-torch- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - - run: pip install --upgrade pip - - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm] - - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html - - run: pip install https://github.com/kpu/kenlm/archive/master.zip - - save_cache: - key: v0.5-torch-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch tests/pipelines | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_pipelines_tf: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-tf-{{ checksum "setup.py" }} - - v0.5-tf- - - run: pip install --upgrade pip - - run: pip install .[sklearn,tf-cpu,testing,sentencepiece] - - run: pip install tensorflow_probability - - save_cache: - key: v0.5-tf-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf tests/pipelines | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_custom_tokenizers: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - RUN_CUSTOM_TOKENIZERS: yes - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-custom_tokenizers-{{ checksum "setup.py" }} - - v0.5-custom_tokenizers- - - run: sudo apt-get -y update && sudo apt-get install -y cmake - - run: - name: install jumanpp - command: | - wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz - tar xvf jumanpp-2.0.0-rc3.tar.xz - mkdir jumanpp-2.0.0-rc3/bld - cd jumanpp-2.0.0-rc3/bld - sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local - sudo make install - - run: pip install --upgrade pip - - run: pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba] - - run: python -m unidic download - - save_cache: - key: v0.5-custom_tokenizers-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest --max-worker-restart=0 -s --make-reports=tests_custom_tokenizers ./tests/models/bert_japanese/test_tokenization_bert_japanese.py ./tests/models/openai/test_tokenization_openai.py ./tests/models/clip/test_tokenization_clip.py | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_examples_torch: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/examples_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-torch_examples-{{ checksum "setup.py" }} - - v0.5-torch_examples- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - - run: pip install --upgrade pip - - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech] - - run: pip install -r examples/pytorch/_tests_requirements.txt - - save_cache: - key: v0.5-torch_examples-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee tests_output.txt - - store_artifacts: - path: ~/transformers/examples_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_examples_tensorflow: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/examples_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-tensorflow_examples-{{ checksum "setup.py" }} - - v0.5-tensorflow_examples- - - run: pip install --upgrade pip - - run: pip install .[sklearn,tensorflow,sentencepiece,testing] - - run: pip install -r examples/tensorflow/_tests_requirements.txt - - save_cache: - key: v0.5-tensorflow_examples-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_tensorflow ./examples/tensorflow/ | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tensorflow_examples_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_examples_flax: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/examples_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-flax_examples-{{ checksum "setup.py" }} - - v0.5-flax_examples- - - run: pip install --upgrade pip - - run: pip install .[flax,testing,sentencepiece] - - run: pip install -r examples/flax/_tests_requirements.txt - - save_cache: - key: v0.5-flax_examples-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_flax ./examples/flax/ | tee tests_output.txt - - store_artifacts: - path: ~/transformers/flax_examples_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_hub: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - HUGGINGFACE_CO_STAGING: yes - RUN_GIT_LFS_TESTS: yes - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-hub-{{ checksum "setup.py" }} - - v0.5-hub- - - run: sudo apt-get -y update && sudo apt-get install git-lfs - - run: | - git config --global user.email "ci@dummy.com" - git config --global user.name "ci" - - run: pip install --upgrade pip - - run: pip install .[torch,sentencepiece,testing] - - save_cache: - key: v0.5-hub-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest --max-worker-restart=0 -sv --make-reports=tests_hub $(cat test_preparation/filtered_test_list.txt) -m is_staging_test | tee tests_output.txt - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - - run_tests_onnxruntime: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/test_preparation - - run: | - if [ ! -s test_preparation/filtered_test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-onnx-{{ checksum "setup.py" }} - - v0.5-onnx- - - run: pip install --upgrade pip - - run: pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba] - - save_cache: - key: v0.5-onnx-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_onnx $(cat test_preparation/filtered_test_list.txt) -k onnx | tee tests_output.txt - - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports + path: test_preparation/generated_config.txt + - continuation/continue: + configuration_path: test_preparation/generated_config.yml check_code_quality: working_directory: ~/transformers @@ -641,7 +98,7 @@ jobs: - restore_cache: keys: - v0.5-code_quality-{{ checksum "setup.py" }} - - v0.5-code_quality- + - v0.5-code-quality - run: pip install --upgrade pip - run: pip install .[all,quality] - save_cache: @@ -670,7 +127,7 @@ jobs: - restore_cache: keys: - v0.5-repository_consistency-{{ checksum "setup.py" }} - - v0.5-repository_consistency- + - v0.5-repository_consistency - run: pip install --upgrade pip - run: pip install .[all,quality] - save_cache: @@ -687,196 +144,19 @@ jobs: - run: python utils/tests_fetcher.py --sanity_check - run: python utils/update_metadata.py --check-only - run_tests_layoutlmv2_and_v3: - working_directory: ~/transformers - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - PYTEST_TIMEOUT: 120 - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - attach_workspace: - at: ~/transformers/filtered_test_list.txt - - run: | - if [ ! -s test_preparation/test_list.txt ]; then - echo "No tests to run, exiting early!" - circleci-agent step halt - fi - - restore_cache: - keys: - - v0.5-torch-{{ checksum "setup.py" }} - - v0.5-torch- - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev - - run: pip install --upgrade pip - - run: pip install .[torch,testing,vision] - - run: pip install torchvision - # The commit `36a65a0907d90ed591479b2ebaa8b61cfa0b4ef0` in `detectron2` break things. - # See https://github.com/facebookresearch/detectron2/commit/36a65a0907d90ed591479b2ebaa8b61cfa0b4ef0#comments. - # TODO: Revert this change back once the above issue is fixed. - - run: python -m pip install 'git+https://github.com/facebookresearch/detectron2.git' - - run: sudo apt install tesseract-ocr - - run: pip install pytesseract - - save_cache: - key: v0.5-torch-{{ checksum "setup.py" }} - paths: - - '~/.cache/pip' - - run: python -m pytest -n 1 --max-worker-restart=0 tests/models/*layoutlmv* --dist=loadfile -s --make-reports=tests_layoutlmv2_and_v3 --durations=100 - - store_artifacts: - path: ~/transformers/tests_output.txt - - store_artifacts: - path: ~/transformers/reports - -# TPU JOBS - run_examples_tpu: - docker: - - image: cimg/python:3.7.12 - environment: - OMP_NUM_THREADS: 1 - TRANSFORMERS_IS_CI: yes - resource_class: xlarge - parallelism: 1 - steps: - - checkout - - go/install - - *checkout_ml_testing - - gcp-gke/install - - gcp-gke/update-kubeconfig-with-credentials: - cluster: $GKE_CLUSTER - perform-login: true - - setup_remote_docker - - *build_push_docker - - *deploy_cluster - - cleanup-gke-jobs: - docker: - - image: cimg/python:3.7.12 - steps: - - gcp-gke/install - - gcp-gke/update-kubeconfig-with-credentials: - cluster: $GKE_CLUSTER - perform-login: true - - *delete_gke_jobs - -workflow_filters: &workflow_filters - filters: - branches: - only: - - main workflows: version: 2 - build_and_test: + setup_and_quality: + when: + not: <> jobs: - check_code_quality - check_repository_consistency - fetch_tests - - run_examples_torch: - requires: - - fetch_tests - - run_examples_tensorflow: - requires: - - fetch_tests - - run_examples_flax: - requires: - - fetch_tests - - run_tests_custom_tokenizers: - requires: - - fetch_tests - - run_tests_torch_and_tf: - requires: - - fetch_tests - - run_tests_torch_and_flax: - requires: - - fetch_tests - - run_tests_torch: - requires: - - fetch_tests - - run_tests_tf: - requires: - - fetch_tests - - run_tests_flax: - requires: - - fetch_tests - - run_tests_pipelines_torch: - requires: - - fetch_tests - - run_tests_pipelines_tf: - requires: - - fetch_tests - - run_tests_onnxruntime: - requires: - - fetch_tests - - run_tests_hub: - requires: - - fetch_tests - - run_tests_layoutlmv2_and_v3: - requires: - - fetch_tests - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - main - jobs: - - fetch_all_tests - - run_examples_torch: - requires: - - fetch_all_tests - - run_examples_tensorflow: - requires: - - fetch_all_tests - - run_examples_flax: - requires: - - fetch_all_tests - - run_tests_custom_tokenizers: - requires: - - fetch_all_tests - - run_tests_torch_and_tf: - requires: - - fetch_all_tests - - run_tests_torch_and_flax: - requires: - - fetch_all_tests - - run_tests_torch: - requires: - - fetch_all_tests - - run_tests_tf: - requires: - - fetch_all_tests - - run_tests_flax: - requires: - - fetch_all_tests - - run_tests_pipelines_torch: - requires: - - fetch_all_tests - - run_tests_pipelines_tf: - requires: - - fetch_all_tests - - run_tests_onnxruntime: - requires: - - fetch_all_tests - - run_tests_hub: - requires: - - fetch_all_tests - - run_tests_layoutlmv2_and_v3: - requires: - - fetch_all_tests -# tpu_testing_jobs: -# triggers: -# - schedule: -# # Set to run at the first minute of every hour. -# cron: "0 8 * * *" -# filters: -# branches: -# only: -# - main -# jobs: -# - cleanup-gke-jobs -# - run_examples_tpu + nightly: + when: <> + jobs: + - check_code_quality + - check_repository_consistency + - fetch_all_tests \ No newline at end of file diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py new file mode 100644 index 0000000000..128283a178 --- /dev/null +++ b/.circleci/create_circleci_config.py @@ -0,0 +1,391 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import os +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import yaml + + +COMMON_ENV_VARIABLES = {"OMP_NUM_THREADS": 1, "TRANSFORMERS_IS_CI": True, "PYTEST_TIMEOUT": 120} +COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "s": None} +DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.7.12"}] +TORCH_SCATTER_INSTALL = "pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html" + + +@dataclass +class CircleCIJob: + name: str + additional_env: Dict[str, Any] = None + cache_name: str = None + cache_version: str = "0.5" + docker_image: List[Dict[str, str]] = None + install_steps: List[str] = None + marker: Optional[str] = None + parallelism: Optional[int] = 1 + pytest_num_workers: int = 8 + pytest_options: Dict[str, Any] = None + resource_class: Optional[str] = "xlarge" + tests_to_run: Optional[List[str]] = None + working_directory: str = "~/transformers" + + def __post_init__(self): + # Deal with defaults for mutable attributes. + if self.additional_env is None: + self.additional_env = {} + if self.cache_name is None: + self.cache_name = self.name + if self.docker_image is None: + # Let's avoid changing the default list and make a copy. + self.docker_image = copy.deepcopy(DEFAULT_DOCKER_IMAGE) + if self.install_steps is None: + self.install_steps = [] + if self.pytest_options is None: + self.pytest_options = {} + if isinstance(self.tests_to_run, str): + self.tests_to_run = [self.tests_to_run] + + def to_dict(self): + job = { + "working_directory": self.working_directory, + "docker": self.docker_image, + "environment": {**COMMON_ENV_VARIABLES, **self.additional_env}, + } + if self.resource_class is not None: + job["resource_class"] = self.resource_class + if self.parallelism is not None: + job["parallelism"] = self.parallelism + steps = [ + "checkout", + {"attach_workspace": {"at": "~/transformers/test_preparation"}}, + { + "restore_cache": { + "keys": [ + f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}', + f"v{self.cache_version}-{self.cache_name}-", + ] + } + }, + ] + steps.extend([{"run": l} for l in self.install_steps]) + steps.append( + { + "save_cache": { + "key": f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}', + "paths": ["~/.cache/pip"], + } + } + ) + + all_options = {**COMMON_PYTEST_OPTIONS, **self.pytest_options} + pytest_flags = [f"--{key}={value}" if value is not None else f"-{key}" for key, value in all_options.items()] + pytest_flags.append( + f"--make-reports={self.name}" if "examples" in self.name else f"--make-reports=tests_{self.name}" + ) + test_command = f"python -m pytest -n {self.pytest_num_workers} " + " ".join(pytest_flags) + if self.tests_to_run is None: + test_command += " << pipeline.parameters.tests_to_run >>" + else: + test_command += " " + " ".join(self.tests_to_run) + if self.marker is not None: + test_command += f" -m {self.marker}" + test_command += " | tee tests_output.txt" + steps.append({"run": {"name": "Run tests", "command": test_command}}) + steps.append({"store_artifacts": {"path": "~/transformers/tests_output.txt"}}) + steps.append({"store_artifacts": {"path": "~/transformers/reports"}}) + job["steps"] = steps + return job + + @property + def job_name(self): + return self.name if "examples" in self.name else f"tests_{self.name}" + + +# JOBS +torch_and_tf_job = CircleCIJob( + "torch_and_tf", + additional_env={"RUN_PT_TF_CROSS_TESTS": True}, + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs", + "git lfs install", + "pip install --upgrade pip", + "pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]", + TORCH_SCATTER_INSTALL, + "pip install tensorflow_probability", + "pip install https://github.com/kpu/kenlm/archive/master.zip", + "pip install git+https://github.com/huggingface/accelerate", + ], + marker="is_pt_tf_cross_test", + pytest_options={"rA": None, "durations": 0}, +) + + +torch_and_flax_job = CircleCIJob( + "torch_and_flax", + additional_env={"RUN_PT_FLAX_CROSS_TESTS": True}, + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", + "pip install --upgrade pip", + "pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]", + TORCH_SCATTER_INSTALL, + "pip install https://github.com/kpu/kenlm/archive/master.zip", + "pip install git+https://github.com/huggingface/accelerate", + ], + marker="is_pt_flax_cross_test", + pytest_options={"rA": None, "durations": 0}, +) + + +torch_job = CircleCIJob( + "torch", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time", + "pip install --upgrade pip", + "pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]", + TORCH_SCATTER_INSTALL, + "pip install https://github.com/kpu/kenlm/archive/master.zip", + "pip install git+https://github.com/huggingface/accelerate", + ], + pytest_num_workers=3, +) + + +tf_job = CircleCIJob( + "tf", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", + "pip install --upgrade pip", + "pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]", + "pip install tensorflow_probability", + "pip install https://github.com/kpu/kenlm/archive/master.zip", + ], + pytest_options={"rA": None}, +) + + +flax_job = CircleCIJob( + "flax", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", + "pip install --upgrade pip", + "pip install .[flax,testing,sentencepiece,flax-speech,vision]", + "pip install https://github.com/kpu/kenlm/archive/master.zip", + ], + pytest_options={"rA": None}, +) + + +pipelines_torch_job = CircleCIJob( + "pipelines_torch", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", + "pip install --upgrade pip", + "pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]", + TORCH_SCATTER_INSTALL, + "pip install https://github.com/kpu/kenlm/archive/master.zip", + ], + pytest_options={"rA": None}, + tests_to_run="tests/pipelines/" +) + + +pipelines_tf_job = CircleCIJob( + "pipelines_tf", + install_steps=[ + "pip install --upgrade pip", + "pip install .[sklearn,tf-cpu,testing,sentencepiece]", + "pip install tensorflow_probability", + ], + pytest_options={"rA": None}, + tests_to_run="tests/pipelines/" +) + + +custom_tokenizers_job = CircleCIJob( + "custom_tokenizers", + additional_env={"RUN_CUSTOM_TOKENIZERS": True}, + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y cmake", + { + "name": "install jumanpp", + "command": + "wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz\n" + "tar xvf jumanpp-2.0.0-rc3.tar.xz\n" + "mkdir jumanpp-2.0.0-rc3/bld\n" + "cd jumanpp-2.0.0-rc3/bld\n" + "sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n" + "sudo make install\n", + }, + "pip install --upgrade pip", + "pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]", + "python -m unidic download", + ], + parallelism=None, + resource_class=None, + tests_to_run=[ + "./tests/models/bert_japanese/test_tokenization_bert_japanese.py", + "./tests/models/openai/test_tokenization_openai.py", + "./tests/models/clip/test_tokenization_clip.py", + ], +) + + +examples_torch_job = CircleCIJob( + "examples_torch", + cache_name="torch_examples", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", + "pip install --upgrade pip", + "pip install .[sklearn,torch,sentencepiece,testing,torch-speech]", + "pip install -r examples/pytorch/_tests_requirements.txt", + ], + tests_to_run="./examples/pytorch/", +) + + +examples_tensorflow_job = CircleCIJob( + "examples_tensorflow", + cache_name="tensorflow_examples", + install_steps=[ + "pip install --upgrade pip", + "pip install .[sklearn,tensorflow,sentencepiece,testing]", + "pip install -r examples/tensorflow/_tests_requirements.txt", + ], + tests_to_run="./examples/tensorflow/", +) + + +examples_flax_job = CircleCIJob( + "examples_flax", + cache_name="flax_examples", + install_steps=[ + "pip install --upgrade pip", + "pip install .[flax,testing,sentencepiece]", + "pip install -r examples/flax/_tests_requirements.txt", + ], + tests_to_run="./examples/flax/", +) + + +hub_job = CircleCIJob( + "hub", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install git-lfs", + 'git config --global user.email "ci@dummy.com"', + 'git config --global user.name "ci"', + "pip install --upgrade pip", + "pip install .[torch,sentencepiece,testing]", + ], + marker="is_staging_test", + pytest_num_workers=1, +) + + +onnx_job = CircleCIJob( + "onnx", + install_steps=[ + "pip install --upgrade pip", + "pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]", + ], + pytest_options={"k onnx": None}, + pytest_num_workers=1, +) + + +layoutlm_job = CircleCIJob( + "layoutlmv2_and_v3", + install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev", + "pip install --upgrade pip", + "pip install .[torch,testing,vision]", + "pip install torchvision", + "pip install 'git+https://github.com/facebookresearch/detectron2.git'", + "sudo apt install tesseract-ocr", + "pip install pytesseract", + ], + tests_to_run="tests/models/*layoutlmv*", + pytest_num_workers=1, + pytest_options={"durations": 100}, +) + + +REGULAR_TESTS = [ + torch_and_tf_job, + torch_and_flax_job, + torch_job, + tf_job, + flax_job, + custom_tokenizers_job, + hub_job, + onnx_job, + layoutlm_job, +] +EXAMPLES_TESTS = [ + examples_torch_job, + examples_tensorflow_job, + examples_flax_job, +] +PIPELINE_TESTS = [ + pipelines_torch_job, + pipelines_tf_job, +] + + +def create_circleci_config(folder=None): + if folder is None: + folder = os.getcwd() + jobs = [] + all_test_file = os.path.join(folder, "test_list.txt") + if os.path.exists(all_test_file): + with open(all_test_file) as f: + all_test_list = f.read() + else: + all_test_list = [] + if len(all_test_list) > 0: + jobs.extend(PIPELINE_TESTS) + + test_file = os.path.join(folder, "filtered_test_list.txt") + if os.path.exists(test_file): + with open(test_file) as f: + test_list = f.read() + else: + test_list = [] + if len(test_list) > 0: + jobs.extend(REGULAR_TESTS) + + example_file = os.path.join(folder, "examples_test_list.txt") + if os.path.exists(example_file) and os.path.getsize(example_file) > 0: + jobs.extend(EXAMPLES_TESTS) + + if len(jobs) > 0: + config = {"version": "2.1"} + config["parameters"] = {"tests_to_run": {"type": "string", "default": test_list}} + config["jobs"] = {j.job_name: j.to_dict() for j in jobs} + config["workflows"] = {"version": 2, "run_tests": {"jobs": [j.job_name for j in jobs]}} + with open(os.path.join(folder, "generated_config.yml"), "w") as f: + f.write(yaml.dump(config, indent=2, width=1000000, sort_keys=False)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--fetcher_folder", type=str, default=None, help="Only test that all tests and modules are accounted for." + ) + args = parser.parse_args() + + create_circleci_config(args.fetcher_folder)