Include all optional dependencies in extras.

Take advantage of this to simplify the Circle CI configuration.

Don't bother with tensorboardX: it's a fallback for PyTorch < 1.1.0.

.circleci/config.yml

@@ -10,11 +10,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install torch
+            - run: sudo pip install .[sklearn,tf,torch,testing]
-            - run: sudo pip install tensorflow
+            - run: sudo pip install codecov pytest-cov
-            - run: sudo pip install --progress-bar off .
-            - run: sudo pip install pytest codecov pytest-cov pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_torch:
@@ -27,10 +24,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install torch
+            - run: sudo pip install .[sklearn,torch,testing]
-            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install codecov pytest-cov
-            - run: sudo pip install pytest codecov pytest-cov pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_tf:
@@ -43,10 +38,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install tensorflow
+            - run: sudo pip install .[sklearn,tf,testing]
-            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install codecov pytest-cov
-            - run: sudo pip install pytest codecov pytest-cov pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_custom_tokenizers:
@@ -55,9 +48,7 @@ jobs:
             - image: circleci/python:3.5
         steps:
             - checkout
-            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install .[mecab,testing]
-            - run: sudo pip install pytest pytest-xdist
-            - run: sudo pip install mecab-python3
             - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
     run_examples_torch:
         working_directory: ~/transformers
@@ -69,10 +60,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install torch
+            - run: sudo pip install .[sklearn,torch,testing]
-            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install -r examples/requirements.txt
-            - run: sudo pip install pytest pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/
     deploy_doc:
         working_directory: ~/transformers
@@ -83,8 +72,7 @@ jobs:
                   fingerprints:
                       - "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
             - checkout
-            - run: sudo pip install --progress-bar off -r docs/requirements.txt
+            - run: sudo pip install .[tf,torch,docs]
-            - run: sudo pip install --progress-bar off .
             - run: ./.circleci/deploy.sh
     check_code_quality:
         working_directory: ~/transformers
@@ -94,9 +82,9 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install --editable .
+            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
-            - run: sudo pip install torch tensorflow
+            - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
-            - run: sudo pip install black git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort flake8
+            - run: sudo pip install .[tf,torch,quality]
             - run: black --check --line-length 119 examples templates tests src utils
             - run: isort --check-only --recursive examples templates tests src utils
             - run: flake8 examples templates tests src utils

CONTRIBUTING.md

@@ -120,7 +120,7 @@ Follow these steps to start contributing:
 4. Set up a development environment by running the following command in a virtual environment:
 
    ```bash
-   $ pip install -r requirements-dev.txt
+   $ pip install -e .[dev]
    ```
 
 5. Develop the features on your branch. Add changed files using `git add` and

setup.cfg

@@ -7,6 +7,7 @@ known_third_party =
     fairseq
     fastprogress
     git
+    MeCab
     nltk
     packaging
     PIL

setup.py

@@ -37,12 +37,20 @@ To create the package for pypi.
 from setuptools import find_packages, setup
 
 
-extras = {
+extras = {}
-    "serving": ["pydantic", "uvicorn", "fastapi"],
+
-    "serving-tf": ["pydantic", "uvicorn", "fastapi", "tensorflow"],
+extras["mecab"] = ["mecab-python3"]
-    "serving-torch": ["pydantic", "uvicorn", "fastapi", "torch"],
+extras["sklearn"] = ["scikit-learn"]
-}
+extras["tf"] = ["tensorflow"]
-extras["all"] = [package for package in extras.values()]
+extras["torch"] = ["torch"]
+
+extras["serving"] = ["pydantic", "uvicorn", "fastapi"]
+extras["all"] = extras["serving"] + ["tensorflow", "torch"]
+
+extras["testing"] = ["pytest", "pytest-xdist"]
+extras["quality"] = ["black", "isort", "flake8"]
+extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
+extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]
 
 setup(
     name="transformers",