SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Roll out the test fetcher on push tests (#13055)

Browse Source 
* Use test fetcher for push tests as well

* Force diff with last commit for circleCI on master

* Fix syntax error

* Style

* Schedule nightly tests
This commit is contained in:
Sylvain Gugger
2021-08-10 14:54:52 +02:00
committed by GitHub
parent 2e0d767ab2
commit 9e9b8f1d99
3 changed files with 421 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

297
.circleci/config.yml
View File

@@ -98,6 +98,37 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_torch_and_tf_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.6
environment:
OMP_NUM_THREADS: 1
RUN_PT_TF_CROSS_TESTS: yes
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-torch_and_tf-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
- run: pip install --upgrade pip
- run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,speech,vision]
- run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html
- save_cache:
key: v0.4-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf tests -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_torch_and_flax: run_tests_torch_and_flax:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -134,6 +165,37 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_torch_and_flax_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.6
environment:
OMP_NUM_THREADS: 1
RUN_PT_FLAX_CROSS_TESTS: yes
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-torch_and_flax-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
- run: pip install --upgrade pip
- run: pip install .[sklearn,flax,torch,testing,sentencepiece,speech,vision]
- run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html
- save_cache:
key: v0.4-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax tests -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_torch: run_tests_torch:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -169,6 +231,36 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_torch_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
OMP_NUM_THREADS: 1
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-torch-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
- run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing,sentencepiece,speech,vision,timm]
- run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html
- save_cache:
key: v0.4-torch-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 3 --dist=loadfile -s --make-reports=tests_torch tests | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_tf: run_tests_tf:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -202,6 +294,34 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_tf_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
OMP_NUM_THREADS: 1
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-tf-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
- save_cache:
key: v0.4-tf-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf tests | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_flax: run_tests_flax:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -235,6 +355,34 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_flax_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
OMP_NUM_THREADS: 1
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-flax-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: sudo pip install .[flax,testing,sentencepiece]
- save_cache:
key: v0.4-flax-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax tests | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_pipelines_torch: run_tests_pipelines_torch:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -271,6 +419,37 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_pipelines_torch_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
OMP_NUM_THREADS: 1
RUN_PIPELINE_TESTS: yes
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-torch-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
- run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing,sentencepiece,speech,vision]
- run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.9.0+cpu.html
- save_cache:
key: v0.4-torch-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test tests | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_pipelines_tf: run_tests_pipelines_tf:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -305,6 +484,35 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_pipelines_tf_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
OMP_NUM_THREADS: 1
RUN_PIPELINE_TESTS: yes
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-tf-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
- save_cache:
key: v0.4-tf-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf $(cat test_list.txt) -m tests | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_custom_tokenizers: run_tests_custom_tokenizers:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -399,8 +607,45 @@ jobs:
path: ~/transformers/test_preparation.txt path: ~/transformers/test_preparation.txt
- run: | - run: |
if [ -f test_list.txt ]; then if [ -f test_list.txt ]; then
python -m pytest -sv $(cat test_list.txt) -m is_staging_test python -m pytest -sv --make-reports=tests_hub $(cat test_list.txt) -m is_staging_test | tee tests_output.txt
fi fi
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_hub_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
HUGGINGFACE_CO_STAGING: yes
RUN_GIT_LFS_TESTS: yes
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-hub-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: sudo apt-get install git-lfs
- run: |
git config --global user.email "ci@dummy.com"
git config --global user.name "ci"
- run: pip install --upgrade pip
- run: pip install .[torch,sentencepiece,testing]
- save_cache:
key: v0.4-hub-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -sv --make-reports=tests_hub tests -m is_staging_test | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
run_tests_onnxruntime: run_tests_onnxruntime:
working_directory: ~/transformers working_directory: ~/transformers
@@ -428,12 +673,41 @@ jobs:
path: ~/transformers/test_preparation.txt path: ~/transformers/test_preparation.txt
- run: | - run: |
if [ -f test_list.txt ]; then if [ -f test_list.txt ]; then
python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch $(cat test_list.txt) -k onnx | tee tests_output.txt python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_onnx $(cat test_list.txt) -k onnx | tee tests_output.txt
fi fi
- store_artifacts: - store_artifacts:
path: ~/transformers/tests_output.txt path: ~/transformers/tests_output.txt
- store_artifacts: - store_artifacts:
path: ~/transformers/reports path: ~/transformers/reports
run_tests_onnxruntime_all:
working_directory: ~/transformers
docker:
- image: circleci/python:3.7
environment:
OMP_NUM_THREADS: 1
TRANSFORMERS_IS_CI: yes
resource_class: xlarge
parallelism: 1
steps:
- checkout
- restore_cache:
keys:
- v0.4-torch-{{ checksum "setup.py" }}
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[torch,testing,sentencepiece,onnxruntime]
- save_cache:
key: v0.4-onnx-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- run: |
python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_onnx tests -k onnx | tee tests_output.txt
- store_artifacts:
path: ~/transformers/tests_output.txt
- store_artifacts:
path: ~/transformers/reports
build_doc: build_doc:
working_directory: ~/transformers working_directory: ~/transformers
docker: docker:
@@ -579,6 +853,25 @@ workflows:
- run_tests_hub - run_tests_hub
- build_doc - build_doc
- deploy_doc: *workflow_filters - deploy_doc: *workflow_filters
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
jobs:
- run_tests_torch_and_tf_all
- run_tests_torch_and_flax_all
- run_tests_torch_all
- run_tests_tf_all
- run_tests_flax_all
- run_tests_pipelines_torch_all
- run_tests_pipelines_tf_all
- run_tests_onnxruntime_all
- run_tests_hub_all
# tpu_testing_jobs: # tpu_testing_jobs:
# triggers: # triggers:
# - schedule: # - schedule:

84
.github/workflows/self-push.yml vendored
View File

@@ -47,9 +47,21 @@ jobs:
python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
- name: Fetch the tests to run
run: |
python utils/tests_fetcher.py | tee test_preparation.txt
- name: Report fetched tests
uses: actions/upload-artifact@v2
with:
name: test_fetched
path: test_preparation.txt
- name: Run all non-slow tests on GPU - name: Run all non-slow tests on GPU
run: | run: |
python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_gpu tests if [ -f test_list.txt ]; then
python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_gpu $(cat test_list.txt)
fi
- name: Failure short reports - name: Failure short reports
if: ${{ always() }} if: ${{ always() }}
@@ -86,12 +98,24 @@ jobs:
# TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
# TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
# #
# - name: Fetch the tests to run
# run: |
# python utils/tests_fetcher.py | tee test_preparation.txt
#
# - name: Report fetched tests
# uses: actions/upload-artifact@v2
# with:
# name: test_fetched
# path: test_preparation.txt
#
# - name: Run all non-slow tests on GPU # - name: Run all non-slow tests on GPU
# env: # env:
# TF_NUM_INTRAOP_THREADS: 8 # TF_NUM_INTRAOP_THREADS: 8
# TF_NUM_INTEROP_THREADS: 1 # TF_NUM_INTEROP_THREADS: 1
# run: | # run: |
# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_gpu tests # if [ -f test_list.txt ]; then
# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_gpu $(cat test_list.txt)
# fi
# #
# - name: Failure short reports # - name: Failure short reports
# if: ${{ always() }} # if: ${{ always() }}
@@ -131,11 +155,23 @@ jobs:
python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
- name: Fetch the tests to run
run: |
python utils/tests_fetcher.py | tee test_preparation.txt
- name: Report fetched tests
uses: actions/upload-artifact@v2
with:
name: test_fetched
path: test_preparation.txt
- name: Run all non-slow tests on GPU - name: Run all non-slow tests on GPU
env: env:
MKL_SERVICE_FORCE_INTEL: 1 MKL_SERVICE_FORCE_INTEL: 1
run: | run: |
python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_multi_gpu tests if [ -f test_list.txt ]; then
python -m pytest -n 2 --dist=loadfile -v --make-reports=tests_torch_multi_gpu $(cat test_list.txt)
fi
- name: Failure short reports - name: Failure short reports
if: ${{ always() }} if: ${{ always() }}
@@ -172,12 +208,24 @@ jobs:
# TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
# TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" # TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
# #
# - name: Fetch the tests to run
# run: |
# python utils/tests_fetcher.py | tee test_preparation.txt
#
# - name: Report fetched tests
# uses: actions/upload-artifact@v2
# with:
# name: test_fetched
# path: test_preparation.txt
#
# - name: Run all non-slow tests on GPU # - name: Run all non-slow tests on GPU
# env: # env:
# TF_NUM_INTRAOP_THREADS: 8 # TF_NUM_INTRAOP_THREADS: 8
# TF_NUM_INTEROP_THREADS: 1 # TF_NUM_INTEROP_THREADS: 1
# run: | # run: |
# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_multi_gpu tests # if [ -f test_list.txt ]; then
# python -m pytest -n 1 --dist=loadfile --make-reports=tests_tf_multi_gpu $(cat test_list.txt)
# fi
# #
# - name: Failure short reports # - name: Failure short reports
# if: ${{ always() }} # if: ${{ always() }}
@@ -216,9 +264,21 @@ jobs:
python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
- name: Fetch the tests to run
run: |
python utils/tests_fetcher.py --filters tests/deepspeed tests/extended | tee test_preparation.txt
- name: Report fetched tests
uses: actions/upload-artifact@v2
with:
name: test_fetched
path: test_preparation.txt
- name: Run all tests on GPU - name: Run all tests on GPU
run: | run: |
python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended if [ -f test_list.txt ]; then
python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_gpu $(cat test_list.txt)
fi
- name: Failure short reports - name: Failure short reports
if: ${{ always() }} if: ${{ always() }}
@@ -257,9 +317,21 @@ jobs:
python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())" python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())" python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
- name: Fetch the tests to run
run: |
python utils/tests_fetcher.py --filters tests/deepspeed tests/extended | tee test_preparation.txt
- name: Report fetched tests
uses: actions/upload-artifact@v2
with:
name: test_fetched
path: test_preparation.txt
- name: Run all tests on GPU - name: Run all tests on GPU
run: | run: |
python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_multi_gpu tests/deepspeed tests/extended if [ -f test_list.txt ]; then
python -m pytest -n 1 --dist=loadfile -v --make-reports=tests_torch_cuda_extensions_multi_gpu $(cat test_list.txt)
fi
- name: Failure short reports - name: Failure short reports
if: ${{ always() }} if: ${{ always() }}

70
utils/tests_fetcher.py
View File

@@ -82,23 +82,39 @@ def diff_is_docstring_only(repo, branching_point, filename):
return old_content_clean == new_content_clean return old_content_clean == new_content_clean
def get_modified_python_files(): def get_modified_python_files(diff_with_last_commit=False):
""" """
Return a list of python files that have been modified between the current head and the master branch. Return a list of python files that have been modified between:
- the current head and the master branch if `diff_with_last_commit=False` (default)
- the current head and its parent commit otherwise.
""" """
repo = Repo(PATH_TO_TRANFORMERS) repo = Repo(PATH_TO_TRANFORMERS)
print(f"Master is at {repo.refs.master.commit}") if not diff_with_last_commit:
print(f"Current head is at {repo.head.commit}") print(f"Master is at {repo.refs.master.commit}")
print(f"Current head is at {repo.head.commit}")
branching_commits = repo.merge_base(repo.refs.master, repo.head) branching_commits = repo.merge_base(repo.refs.master, repo.head)
for commit in branching_commits: for commit in branching_commits:
print(f"Branching commit: {commit}") print(f"Branching commit: {commit}")
return get_diff(repo, repo.head.commit, branching_commits)
else:
print(f"Master is at {repo.head.commit}")
parent_commits = repo.head.commit.parents
for commit in parent_commits:
print(f"Parent commit: {commit}")
return get_diff(repo, repo.head.commit, parent_commits)
def get_diff(repo, base_commit, commits):
"""
Get's the diff between one or several commits and the head of the repository.
"""
print("\n### DIFF ###\n") print("\n### DIFF ###\n")
code_diff = [] code_diff = []
for commit in branching_commits: for commit in commits:
for diff_obj in commit.diff(repo.head.commit): for diff_obj in commit.diff(base_commit):
# We always add new python files # We always add new python files
if diff_obj.change_type == "A" and diff_obj.b_path.endswith(".py"): if diff_obj.change_type == "A" and diff_obj.b_path.endswith(".py"):
code_diff.append(diff_obj.b_path) code_diff.append(diff_obj.b_path)
@@ -365,8 +381,8 @@ def sanity_check():
) )
def infer_tests_to_run(output_file): def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None):
modified_files = get_modified_python_files() modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit)
print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}") print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}")
# Create the map that will give us all impacted modules. # Create the map that will give us all impacted modules.
@@ -396,6 +412,10 @@ def infer_tests_to_run(output_file):
# Remove duplicates # Remove duplicates
test_files_to_run = sorted(list(set(test_files_to_run))) test_files_to_run = sorted(list(set(test_files_to_run)))
if filters is not None:
for filter in filters:
test_files_to_run = [f for f in test_files_to_run if f.startswith(filter)]
print(f"\n### TEST TO RUN ###\n{_print_list(test_files_to_run)}") print(f"\n### TEST TO RUN ###\n{_print_list(test_files_to_run)}")
if len(test_files_to_run) > 0: if len(test_files_to_run) > 0:
with open(output_file, "w", encoding="utf-8") as f: with open(output_file, "w", encoding="utf-8") as f:
@@ -410,22 +430,28 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
"--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run" "--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run"
) )
parser.add_argument(
"--diff_with_last_commit",
action="store_true",
help="To fetch the tests between the current commit and the last commit",
)
parser.add_argument(
"--filters", type=str, nargs="*", help="Only keep the test files matching one of those filters."
)
args = parser.parse_args() args = parser.parse_args()
if args.sanity_check: if args.sanity_check:
sanity_check() sanity_check()
else: else:
repo = Repo(PATH_TO_TRANFORMERS) repo = Repo(PATH_TO_TRANFORMERS)
# For now we run all tests on the master branch. After testing this more and making sure it works most of the
# time, we will apply the same logic to the tests on the master branch and only run the whole suite once per diff_with_last_commit = args.diff_with_last_commit
# day.
if not repo.head.is_detached and repo.head.ref == repo.refs.master: if not repo.head.is_detached and repo.head.ref == repo.refs.master:
print("Master branch detected, running all tests.") print("Master branch detected, fetching tests against last commit.")
diff_with_last_commit = True
try:
infer_tests_to_run(args.output_file, diff_with_last_commit=diff_with_last_commit, filters=args.filters)
except Exception as e:
print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.")
with open(args.output_file, "w", encoding="utf-8") as f: with open(args.output_file, "w", encoding="utf-8") as f:
f.write("./tests/") f.write("./tests/")
else:
try:
infer_tests_to_run(args.output_file)
except Exception as e:
print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.")
with open(args.output_file, "w", encoding="utf-8") as f:
f.write("./tests/")