diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 0000000000..2c8f906aba
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,11 @@
+version: 2
+jobs:
+    build:
+        working_directory: ~/pytorch-pretrained-BERT
+        docker:
+            - image: circleci/python:3.7
+        steps:
+            - checkout
+            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install pytest
+            - run: python -m pytest -sv tests/
diff --git a/README.md b/README.md
index a487b03fdb..e0c64e36d7 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 # PyTorch Pretrained Bert
 
+[![CircleCI](https://circleci.com/gh/huggingface/pytorch-pretrained-BERT.svg?style=svg)](https://circleci.com/gh/huggingface/pytorch-pretrained-BERT)
+
 This repository contains an op-for-op PyTorch reimplementation of [Google's TensorFlow repository for the BERT model](https://github.com/google-research/bert) that was released together with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
 
 This implementation is provided with [Google's pre-trained models](https://github.com/google-research/bert), examples, notebooks and a command-line interface to load any pre-trained TensorFlow checkpoint for BERT is also provided.