From 29cbab98f0b36a3056e2982bf968c8370bad3838 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:59 +0100 Subject: [PATCH] Parallelize tests on Circle CI. Set the number of CPUs manually based on the Circle CI resource class, or else we're getting 36 CPUs, which is far too much (perhaps that's the underlying hardware and not what Circle CI allocates to us). Don't parallelize the custom tokenizers tests because they take less than one second to run and parallelization actually makes them slower. --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b094067eb5..a7496c81e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,9 +11,9 @@ jobs: - run: sudo pip install torch - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov + - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov - run: codecov build_py3_torch: working_directory: ~/transformers @@ -25,10 +25,10 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: python -m pytest -sv ./examples/ + - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov + - run: python -m pytest -n 8 -s -v ./examples/ - run: codecov build_py3_tf: working_directory: ~/transformers @@ -40,9 +40,9 @@ jobs: - checkout - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov + - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov - run: codecov build_py3_custom_tokenizers: working_directory: ~/transformers @@ -51,7 +51,7 @@ jobs: steps: - checkout - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest + - run: sudo pip install pytest pytest-xdist - run: sudo pip install mecab-python3 - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: