diff --git a/.circleci/config.yml b/.circleci/config.yml
index 35de1281fb..e46529556d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -95,7 +95,7 @@ jobs:
         steps:
             - checkout
             - run: sudo pip install --editable .
-            - run: sudo pip install torch tensorflow tensorboardX scikit-learn
+            - run: sudo pip install torch tensorflow
             - run: sudo pip install black git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort flake8
             - run: black --check --line-length 119 examples templates transformers utils
             - run: isort --check-only --recursive examples templates transformers utils
diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py
index c43f13de0c..53669623b6 100644
--- a/examples/distillation/distiller.py
+++ b/examples/distillation/distiller.py
@@ -19,6 +19,7 @@ import math
 import os
 import time
 
+import psutil
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -27,7 +28,6 @@ from torch.utils.data import BatchSampler, DataLoader, RandomSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm
 
-import psutil
 from grouped_batch_sampler import GroupedBatchSampler, create_lengths_groups
 from lm_seqs_dataset import LmSeqsDataset
 from transformers import get_linear_schedule_with_warmup
diff --git a/examples/distillation/utils.py b/examples/distillation/utils.py
index b081f239c3..211e7c61da 100644
--- a/examples/distillation/utils.py
+++ b/examples/distillation/utils.py
@@ -20,11 +20,10 @@ import logging
 import os
 import socket
 
+import git
 import numpy as np
 import torch
 
-import git
-
 
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s",
diff --git a/examples/mm-imdb/utils_mmimdb.py b/examples/mm-imdb/utils_mmimdb.py
index 7a52a99b1a..aa0460639c 100644
--- a/examples/mm-imdb/utils_mmimdb.py
+++ b/examples/mm-imdb/utils_mmimdb.py
@@ -20,11 +20,10 @@ from collections import Counter
 
 import torch
 import torch.nn as nn
-from torch.utils.data import Dataset
-
 import torchvision
 import torchvision.transforms as transforms
 from PIL import Image
+from torch.utils.data import Dataset
 
 
 POOLING_BREAKDOWN = {1: (1, 1), 2: (2, 1), 3: (3, 1), 4: (2, 2), 5: (5, 1), 6: (3, 2), 7: (7, 1), 8: (4, 2), 9: (3, 3)}
diff --git a/examples/pplm/run_pplm_discrim_train.py b/examples/pplm/run_pplm_discrim_train.py
index 1c21c56c86..44f6b726d8 100644
--- a/examples/pplm/run_pplm_discrim_train.py
+++ b/examples/pplm/run_pplm_discrim_train.py
@@ -26,12 +26,12 @@ import torch
 import torch.nn.functional as F
 import torch.optim as optim
 import torch.utils.data as data
-from tqdm import tqdm, trange
-
 from nltk.tokenize.treebank import TreebankWordDetokenizer
-from pplm_classification_head import ClassificationHead
 from torchtext import data as torchtext_data
 from torchtext import datasets
+from tqdm import tqdm, trange
+
+from pplm_classification_head import ClassificationHead
 from transformers import GPT2LMHeadModel, GPT2Tokenizer
 
 
diff --git a/examples/run_ner.py b/examples/run_ner.py
index 7120c37364..34ba2663bf 100644
--- a/examples/run_ner.py
+++ b/examples/run_ner.py
@@ -25,13 +25,13 @@ import random
 
 import numpy as np
 import torch
+from seqeval.metrics import f1_score, precision_score, recall_score
 from tensorboardX import SummaryWriter
 from torch.nn import CrossEntropyLoss
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-from seqeval.metrics import f1_score, precision_score, recall_score
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py
index 511a98e942..8398ccb4cb 100644
--- a/examples/run_tf_glue.py
+++ b/examples/run_tf_glue.py
@@ -1,8 +1,8 @@
 import os
 
 import tensorflow as tf
-
 import tensorflow_datasets
+
 from transformers import (
     BertConfig,
     BertForSequenceClassification,
diff --git a/examples/run_tf_ner.py b/examples/run_tf_ner.py
index c360f9e778..6aa0f4bc32 100644
--- a/examples/run_tf_ner.py
+++ b/examples/run_tf_ner.py
@@ -9,9 +9,9 @@ import re
 import numpy as np
 import tensorflow as tf
 from absl import app, flags, logging
-
 from fastprogress import master_bar, progress_bar
 from seqeval import metrics
+
 from transformers import (
     TF2_WEIGHTS_NAME,
     BertConfig,
diff --git a/setup.cfg b/setup.cfg
index b3f95d39d3..f59ce55df7 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,7 +3,21 @@ ensure_newline_before_comments = True
 force_grid_wrap = 0
 include_trailing_comma = True
 known_first_party = transformers
-known_third_party = packaging
+known_third_party =
+    fairseq
+    fastprogress
+    git
+    nltk
+    packaging
+    PIL
+    psutil
+    seqeval
+    sklearn
+    tensorboardX
+    tensorflow_datasets
+    torchtext
+    torchvision
+
 line_length = 119
 lines_after_imports = 2
 multi_line_output = 3
diff --git a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py
index 588205737b..7e86f3a933 100644
--- a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py
+++ b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py
@@ -20,12 +20,12 @@ import argparse
 import logging
 import pathlib
 
-import torch
-from packaging import version
-
 import fairseq
+import torch
 from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
 from fairseq.modules import TransformerSentenceEncoderLayer
+from packaging import version
+
 from transformers.modeling_bert import (
     BertConfig,
     BertIntermediate,