diff --git a/examples/seq2seq/finetune_bart_tiny.sh b/examples/seq2seq/finetune_bart_tiny.sh
index b1277d5d7b..b4197e5ac6 100755
--- a/examples/seq2seq/finetune_bart_tiny.sh
+++ b/examples/seq2seq/finetune_bart_tiny.sh
@@ -12,7 +12,7 @@ export OUTPUT_DIR=${CURRENT_DIR}/${OUTPUT_DIR_NAME}
 # Make output directory if it doesn't exist
 mkdir -p $OUTPUT_DIR
 
-# Add parent directory to python path to access lightning_base.py and utils.py
+# Add parent directory to python path to access lightning_base.py and testing_utils.py
 export PYTHONPATH="../":"${PYTHONPATH}"
 python finetune.py \
 --data_dir=cnn_tiny/ \
diff --git a/examples/seq2seq/test_seq2seq_examples.py b/examples/seq2seq/test_seq2seq_examples.py
index bd1b108660..92e75993a9 100644
--- a/examples/seq2seq/test_seq2seq_examples.py
+++ b/examples/seq2seq/test_seq2seq_examples.py
@@ -12,6 +12,7 @@ import torch
 from torch.utils.data import DataLoader
 
 from transformers import AutoTokenizer
+from transformers.testing_utils import require_multigpu
 
 from .distillation import distill_main, evaluate_checkpoint
 from .finetune import main
@@ -107,7 +108,7 @@ class TestSummarizationDistiller(unittest.TestCase):
         logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
         return cls
 
-    @unittest.skipUnless(torch.cuda.device_count() > 1, "skipping multiGPU test")
+    @require_multigpu
     def test_multigpu(self):
         updates = dict(no_teacher=True, freeze_encoder=True, gpus=2, sortish_sampler=False,)
         self._test_distiller_cli(updates)
diff --git a/tests/utils.py b/src/transformers/testing_utils.py
similarity index 100%
rename from tests/utils.py
rename to src/transformers/testing_utils.py
diff --git a/tests/test_activations.py b/tests/test_activations.py
index 79e9eec018..f2c1b2a52d 100644
--- a/tests/test_activations.py
+++ b/tests/test_activations.py
@@ -1,8 +1,7 @@
 import unittest
 
 from transformers import is_torch_available
-
-from .utils import require_torch
+from transformers.testing_utils import require_torch
 
 
 if is_torch_available():
diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py
index efa8a19b05..1d96180f07 100644
--- a/tests/test_benchmark.py
+++ b/tests/test_benchmark.py
@@ -4,8 +4,7 @@ import unittest
 from pathlib import Path
 
 from transformers import AutoConfig, is_torch_available
-
-from .utils import require_torch, torch_device
+from transformers.testing_utils import require_torch, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_benchmark_tf.py b/tests/test_benchmark_tf.py
index 5ef92e8f66..929d5624c4 100644
--- a/tests/test_benchmark_tf.py
+++ b/tests/test_benchmark_tf.py
@@ -4,8 +4,7 @@ import unittest
 from pathlib import Path
 
 from transformers import AutoConfig, is_tf_available
-
-from .utils import require_tf
+from transformers.testing_utils import require_tf
 
 
 if is_tf_available():
diff --git a/tests/test_configuration_auto.py b/tests/test_configuration_auto.py
index 5262be2e7c..e3a66eb85b 100644
--- a/tests/test_configuration_auto.py
+++ b/tests/test_configuration_auto.py
@@ -19,8 +19,7 @@ import unittest
 from transformers.configuration_auto import CONFIG_MAPPING, AutoConfig
 from transformers.configuration_bert import BertConfig
 from transformers.configuration_roberta import RobertaConfig
-
-from .utils import DUMMY_UNKWOWN_IDENTIFIER
+from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER
 
 
 SAMPLE_ROBERTA_CONFIG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/dummy-config.json")
diff --git a/tests/test_doc_samples.py b/tests/test_doc_samples.py
index a88c2ca5fa..5ce7184154 100644
--- a/tests/test_doc_samples.py
+++ b/tests/test_doc_samples.py
@@ -21,8 +21,7 @@ from pathlib import Path
 from typing import List, Union
 
 import transformers
-
-from .utils import require_tf, require_torch, slow
+from transformers.testing_utils import require_tf, require_torch, slow
 
 
 logger = logging.getLogger()
diff --git a/tests/test_modeling_albert.py b/tests/test_modeling_albert.py
index f5c0566bdc..844721f2ea 100644
--- a/tests/test_modeling_albert.py
+++ b/tests/test_modeling_albert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_auto.py b/tests/test_modeling_auto.py
index 0529528c60..cbbf857bc0 100644
--- a/tests/test_modeling_auto.py
+++ b/tests/test_modeling_auto.py
@@ -17,8 +17,7 @@
 import unittest
 
 from transformers import is_torch_available
-
-from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, require_torch, slow
+from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, require_torch, slow
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_bart.py b/tests/test_modeling_bart.py
index 668d250038..4dbfdd1ccd 100644
--- a/tests/test_modeling_bart.py
+++ b/tests/test_modeling_bart.py
@@ -20,10 +20,10 @@ import timeout_decorator  # noqa
 
 from transformers import is_torch_available
 from transformers.file_utils import cached_property
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_bert.py b/tests/test_modeling_bert.py
index 5b2397b0ae..8a689fb843 100644
--- a/tests/test_modeling_bert.py
+++ b/tests/test_modeling_bert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_camembert.py b/tests/test_modeling_camembert.py
index 5d57f862c8..606915de2a 100644
--- a/tests/test_modeling_camembert.py
+++ b/tests/test_modeling_camembert.py
@@ -16,8 +16,7 @@
 import unittest
 
 from transformers import is_torch_available
-
-from .utils import require_torch, slow, torch_device
+from transformers.testing_utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py
index 4df1f75011..0699cf2f67 100644
--- a/tests/test_modeling_common.py
+++ b/tests/test_modeling_common.py
@@ -21,8 +21,7 @@ import unittest
 from typing import List
 
 from transformers import is_torch_available
-
-from .utils import require_multigpu, require_torch, slow, torch_device
+from transformers.testing_utils import require_multigpu, require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_ctrl.py b/tests/test_modeling_ctrl.py
index 3e480ae047..0d6e8f5aee 100644
--- a/tests/test_modeling_ctrl.py
+++ b/tests/test_modeling_ctrl.py
@@ -16,10 +16,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_distilbert.py b/tests/test_modeling_distilbert.py
index 14d505abb6..0c602851ef 100644
--- a/tests/test_modeling_distilbert.py
+++ b/tests/test_modeling_distilbert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_electra.py b/tests/test_modeling_electra.py
index 5af2b6e8f3..4f113bd847 100644
--- a/tests/test_modeling_electra.py
+++ b/tests/test_modeling_electra.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_encoder_decoder.py b/tests/test_modeling_encoder_decoder.py
index 65f479e8f3..e61ef2cee4 100644
--- a/tests/test_modeling_encoder_decoder.py
+++ b/tests/test_modeling_encoder_decoder.py
@@ -18,12 +18,12 @@ import tempfile
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 # TODO(PVP): this line reruns all the tests in BertModelTest; not sure whether this can be prevented
 # for now only run module with pytest tests/test_modeling_encoder_decoder.py::EncoderDecoderModelTest
 from .test_modeling_bert import BertModelTester
 from .test_modeling_common import ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_flaubert.py b/tests/test_modeling_flaubert.py
index 07c76d6ec3..03e4af6ad8 100644
--- a/tests/test_modeling_flaubert.py
+++ b/tests/test_modeling_flaubert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_gpt2.py b/tests/test_modeling_gpt2.py
index 2dbda2e850..71f8759110 100644
--- a/tests/test_modeling_gpt2.py
+++ b/tests/test_modeling_gpt2.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_longformer.py b/tests/test_modeling_longformer.py
index 7579ee38ba..77c3983c34 100644
--- a/tests/test_modeling_longformer.py
+++ b/tests/test_modeling_longformer.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_marian.py b/tests/test_modeling_marian.py
index c66094c447..4c936af503 100644
--- a/tests/test_modeling_marian.py
+++ b/tests/test_modeling_marian.py
@@ -19,8 +19,7 @@ import unittest
 from transformers import is_torch_available
 from transformers.file_utils import cached_property
 from transformers.hf_api import HfApi
-
-from .utils import require_torch, slow, torch_device
+from transformers.testing_utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_mobilebert.py b/tests/test_modeling_mobilebert.py
index 4f086f081e..6d46c319c2 100644
--- a/tests/test_modeling_mobilebert.py
+++ b/tests/test_modeling_mobilebert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_openai.py b/tests/test_modeling_openai.py
index 89dc0dfe0d..7818623672 100644
--- a/tests/test_modeling_openai.py
+++ b/tests/test_modeling_openai.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_reformer.py b/tests/test_modeling_reformer.py
index 7ad651918b..1a3b9ad4c1 100644
--- a/tests/test_modeling_reformer.py
+++ b/tests/test_modeling_reformer.py
@@ -16,10 +16,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_multigpu, require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
-from .utils import require_multigpu, require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_roberta.py b/tests/test_modeling_roberta.py
index 516670b517..e408a6b4a7 100644
--- a/tests/test_modeling_roberta.py
+++ b/tests/test_modeling_roberta.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_t5.py b/tests/test_modeling_t5.py
index 39254c4f47..a75e46a8fd 100644
--- a/tests/test_modeling_t5.py
+++ b/tests/test_modeling_t5.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_tf_albert.py b/tests/test_modeling_tf_albert.py
index cdcf0ffaf3..f59931424b 100644
--- a/tests/test_modeling_tf_albert.py
+++ b/tests/test_modeling_tf_albert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import AlbertConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_auto.py b/tests/test_modeling_tf_auto.py
index 9cf3854f59..3687513dbf 100644
--- a/tests/test_modeling_tf_auto.py
+++ b/tests/test_modeling_tf_auto.py
@@ -17,8 +17,7 @@
 import unittest
 
 from transformers import is_tf_available
-
-from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, require_tf, slow
+from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_bert.py b/tests/test_modeling_tf_bert.py
index 4fab58920b..042f3b4006 100644
--- a/tests/test_modeling_tf_bert.py
+++ b/tests/test_modeling_tf_bert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import BertConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_camembert.py b/tests/test_modeling_tf_camembert.py
index 9a256c84ea..fa962dd615 100644
--- a/tests/test_modeling_tf_camembert.py
+++ b/tests/test_modeling_tf_camembert.py
@@ -16,8 +16,7 @@
 import unittest
 
 from transformers import is_tf_available
-
-from .utils import require_tf, slow
+from transformers.testing_utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py
index 13adcd8984..56fc8c9cf7 100644
--- a/tests/test_modeling_tf_common.py
+++ b/tests/test_modeling_tf_common.py
@@ -23,8 +23,7 @@ import unittest
 from importlib import import_module
 
 from transformers import is_tf_available, is_torch_available
-
-from .utils import _tf_gpu_memory_limit, require_tf
+from transformers.testing_utils import _tf_gpu_memory_limit, require_tf
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_ctrl.py b/tests/test_modeling_tf_ctrl.py
index 789dc65ec4..462a8bb2c2 100644
--- a/tests/test_modeling_tf_ctrl.py
+++ b/tests/test_modeling_tf_ctrl.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import CTRLConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_distilbert.py b/tests/test_modeling_tf_distilbert.py
index 5e42402481..7d7b63fb0e 100644
--- a/tests/test_modeling_tf_distilbert.py
+++ b/tests/test_modeling_tf_distilbert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import DistilBertConfig, is_tf_available
+from transformers.testing_utils import require_tf
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_electra.py b/tests/test_modeling_tf_electra.py
index 225e0e8be2..a0a1cef030 100644
--- a/tests/test_modeling_tf_electra.py
+++ b/tests/test_modeling_tf_electra.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import ElectraConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_flaubert.py b/tests/test_modeling_tf_flaubert.py
index ad7d3d61aa..1b3e6d8823 100644
--- a/tests/test_modeling_tf_flaubert.py
+++ b/tests/test_modeling_tf_flaubert.py
@@ -16,8 +16,7 @@
 import unittest
 
 from transformers import is_tf_available
-
-from .utils import require_tf, slow
+from transformers.testing_utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_gpt2.py b/tests/test_modeling_tf_gpt2.py
index 2f733491a9..7728c8b1f6 100644
--- a/tests/test_modeling_tf_gpt2.py
+++ b/tests/test_modeling_tf_gpt2.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import GPT2Config, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_mobilebert.py b/tests/test_modeling_tf_mobilebert.py
index 5b3c6b820d..e43d0d84cf 100644
--- a/tests/test_modeling_tf_mobilebert.py
+++ b/tests/test_modeling_tf_mobilebert.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import MobileBertConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_openai_gpt.py b/tests/test_modeling_tf_openai_gpt.py
index 5baf108c58..8e32949102 100644
--- a/tests/test_modeling_tf_openai_gpt.py
+++ b/tests/test_modeling_tf_openai_gpt.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import OpenAIGPTConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_roberta.py b/tests/test_modeling_tf_roberta.py
index 72df2ebbbd..65752e994a 100644
--- a/tests/test_modeling_tf_roberta.py
+++ b/tests/test_modeling_tf_roberta.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import RobertaConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_t5.py b/tests/test_modeling_tf_t5.py
index 2f996fc90c..be4637f465 100644
--- a/tests/test_modeling_tf_t5.py
+++ b/tests/test_modeling_tf_t5.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import T5Config, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_transfo_xl.py b/tests/test_modeling_tf_transfo_xl.py
index 438b92c46a..408b3c02b0 100644
--- a/tests/test_modeling_tf_transfo_xl.py
+++ b/tests/test_modeling_tf_transfo_xl.py
@@ -18,10 +18,10 @@ import random
 import unittest
 
 from transformers import TransfoXLConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_xlm.py b/tests/test_modeling_tf_xlm.py
index acaadaf344..26cdb0a39c 100644
--- a/tests/test_modeling_tf_xlm.py
+++ b/tests/test_modeling_tf_xlm.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_xlm_roberta.py b/tests/test_modeling_tf_xlm_roberta.py
index c901eb0525..c27b0576b8 100644
--- a/tests/test_modeling_tf_xlm_roberta.py
+++ b/tests/test_modeling_tf_xlm_roberta.py
@@ -16,8 +16,7 @@
 import unittest
 
 from transformers import is_tf_available
-
-from .utils import require_tf, slow
+from transformers.testing_utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_tf_xlnet.py b/tests/test_modeling_tf_xlnet.py
index f8fee2d45e..ecc498c1d9 100644
--- a/tests/test_modeling_tf_xlnet.py
+++ b/tests/test_modeling_tf_xlnet.py
@@ -18,10 +18,10 @@ import random
 import unittest
 
 from transformers import XLNetConfig, is_tf_available
+from transformers.testing_utils import require_tf, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import require_tf, slow
 
 
 if is_tf_available():
diff --git a/tests/test_modeling_transfo_xl.py b/tests/test_modeling_transfo_xl.py
index 0ec552da07..db552b2798 100644
--- a/tests/test_modeling_transfo_xl.py
+++ b/tests/test_modeling_transfo_xl.py
@@ -17,10 +17,10 @@ import random
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_multigpu, require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_multigpu, require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_xlm.py b/tests/test_modeling_xlm.py
index 52432113ad..e93242f341 100644
--- a/tests/test_modeling_xlm.py
+++ b/tests/test_modeling_xlm.py
@@ -17,10 +17,10 @@
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_xlm_roberta.py b/tests/test_modeling_xlm_roberta.py
index 3d035f48fc..f4098118cb 100644
--- a/tests/test_modeling_xlm_roberta.py
+++ b/tests/test_modeling_xlm_roberta.py
@@ -17,8 +17,7 @@
 import unittest
 
 from transformers import is_torch_available
-
-from .utils import slow
+from transformers.testing_utils import slow
 
 
 if is_torch_available():
diff --git a/tests/test_modeling_xlnet.py b/tests/test_modeling_xlnet.py
index e3701bdc44..99ea832a8b 100644
--- a/tests/test_modeling_xlnet.py
+++ b/tests/test_modeling_xlnet.py
@@ -18,10 +18,10 @@ import random
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import require_torch, slow, torch_device
 
 
 if is_torch_available():
diff --git a/tests/test_onnx.py b/tests/test_onnx.py
index 98431fd491..e9c0c6b48a 100644
--- a/tests/test_onnx.py
+++ b/tests/test_onnx.py
@@ -3,9 +3,9 @@ from os.path import dirname, exists
 from shutil import rmtree
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 
-from tests.utils import require_tf, require_torch, slow
 from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
 from transformers.convert_graph_to_onnx import convert, ensure_valid_input, infer_shapes
+from transformers.testing_utils import require_tf, require_torch, slow
 
 
 class FuncContiguousArgs:
diff --git a/tests/test_optimization.py b/tests/test_optimization.py
index 8c9ebb2dd2..a38e764319 100644
--- a/tests/test_optimization.py
+++ b/tests/test_optimization.py
@@ -19,8 +19,7 @@ import tempfile
 import unittest
 
 from transformers import is_torch_available
-
-from .utils import require_torch
+from transformers.testing_utils import require_torch
 
 
 if is_torch_available():
diff --git a/tests/test_optimization_tf.py b/tests/test_optimization_tf.py
index b4998b7e5b..0eab306a85 100644
--- a/tests/test_optimization_tf.py
+++ b/tests/test_optimization_tf.py
@@ -1,8 +1,7 @@
 import unittest
 
 from transformers import is_tf_available
-
-from .utils import require_tf
+from transformers.testing_utils import require_tf
 
 
 if is_tf_available():
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 88ca2732f5..cfad071bd2 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -3,8 +3,7 @@ from typing import Iterable, List, Optional
 
 from transformers import pipeline
 from transformers.pipelines import SUPPORTED_TASKS, DefaultArgumentHandler, Pipeline
-
-from .utils import require_tf, require_torch, slow, torch_device
+from transformers.testing_utils import require_tf, require_torch, slow, torch_device
 
 
 DEFAULT_DEVICE_NUM = -1 if torch_device == "cpu" else 0
diff --git a/tests/test_tokenization_auto.py b/tests/test_tokenization_auto.py
index bc07117429..54bfb2e13c 100644
--- a/tests/test_tokenization_auto.py
+++ b/tests/test_tokenization_auto.py
@@ -27,10 +27,9 @@ from transformers import (
     RobertaTokenizer,
     RobertaTokenizerFast,
 )
+from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER  # noqa: F401
 from transformers.tokenization_auto import TOKENIZER_MAPPING
 
-from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, slow  # noqa: F401
-
 
 class AutoTokenizerTest(unittest.TestCase):
     # @slow
diff --git a/tests/test_tokenization_bert.py b/tests/test_tokenization_bert.py
index 919d1a72a4..7aa3fbe1d6 100644
--- a/tests/test_tokenization_bert.py
+++ b/tests/test_tokenization_bert.py
@@ -17,6 +17,7 @@
 import os
 import unittest
 
+from transformers.testing_utils import slow
 from transformers.tokenization_bert import (
     VOCAB_FILES_NAMES,
     BasicTokenizer,
@@ -29,7 +30,6 @@ from transformers.tokenization_bert import (
 )
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import slow
 
 
 class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
diff --git a/tests/test_tokenization_bert_japanese.py b/tests/test_tokenization_bert_japanese.py
index 50a9d069b7..03ff26e3a0 100644
--- a/tests/test_tokenization_bert_japanese.py
+++ b/tests/test_tokenization_bert_japanese.py
@@ -17,6 +17,7 @@
 import os
 import unittest
 
+from transformers.testing_utils import custom_tokenizers
 from transformers.tokenization_bert import WordpieceTokenizer
 from transformers.tokenization_bert_japanese import (
     VOCAB_FILES_NAMES,
@@ -26,7 +27,6 @@ from transformers.tokenization_bert_japanese import (
 )
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import custom_tokenizers
 
 
 @custom_tokenizers
diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py
index 92098916ad..d14fec512b 100644
--- a/tests/test_tokenization_common.py
+++ b/tests/test_tokenization_common.py
@@ -22,8 +22,8 @@ import tempfile
 from collections import OrderedDict
 from typing import TYPE_CHECKING, Dict, List, Tuple, Union
 
-from tests.utils import require_tf, require_torch, slow
 from transformers import PreTrainedTokenizer, PreTrainedTokenizerBase, PreTrainedTokenizerFast
+from transformers.testing_utils import require_tf, require_torch, slow
 
 
 if TYPE_CHECKING:
diff --git a/tests/test_tokenization_distilbert.py b/tests/test_tokenization_distilbert.py
index 3a2d12db95..bee28425c7 100644
--- a/tests/test_tokenization_distilbert.py
+++ b/tests/test_tokenization_distilbert.py
@@ -14,10 +14,10 @@
 # limitations under the License.
 
 
+from transformers.testing_utils import slow
 from transformers.tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
 
 from .test_tokenization_bert import BertTokenizationTest
-from .utils import slow
 
 
 class DistilBertTokenizationTest(BertTokenizationTest):
diff --git a/tests/test_tokenization_fast.py b/tests/test_tokenization_fast.py
index 010f29d0fc..e2de9acf03 100644
--- a/tests/test_tokenization_fast.py
+++ b/tests/test_tokenization_fast.py
@@ -3,7 +3,6 @@ import unittest
 from collections import namedtuple
 from itertools import takewhile
 
-from tests.utils import require_torch
 from transformers import (
     BertTokenizer,
     BertTokenizerFast,
@@ -16,6 +15,7 @@ from transformers import (
     TransfoXLTokenizer,
     is_torch_available,
 )
+from transformers.testing_utils import require_torch
 from transformers.tokenization_distilbert import DistilBertTokenizerFast
 from transformers.tokenization_openai import OpenAIGPTTokenizerFast
 from transformers.tokenization_roberta import RobertaTokenizerFast
diff --git a/tests/test_tokenization_roberta.py b/tests/test_tokenization_roberta.py
index c48603203b..f2a0cc7424 100644
--- a/tests/test_tokenization_roberta.py
+++ b/tests/test_tokenization_roberta.py
@@ -18,10 +18,10 @@ import json
 import os
 import unittest
 
+from transformers.testing_utils import slow
 from transformers.tokenization_roberta import VOCAB_FILES_NAMES, AddedToken, RobertaTokenizer, RobertaTokenizerFast
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import slow
 
 
 class RobertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
diff --git a/tests/test_tokenization_transfo_xl.py b/tests/test_tokenization_transfo_xl.py
index aa6b95e2da..f9f56df34b 100644
--- a/tests/test_tokenization_transfo_xl.py
+++ b/tests/test_tokenization_transfo_xl.py
@@ -18,9 +18,9 @@ import os
 import unittest
 
 from transformers import is_torch_available
+from transformers.testing_utils import require_torch
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import require_torch
 
 
 if is_torch_available():
diff --git a/tests/test_tokenization_utils.py b/tests/test_tokenization_utils.py
index fb3f677d9e..2aa3f31123 100644
--- a/tests/test_tokenization_utils.py
+++ b/tests/test_tokenization_utils.py
@@ -17,10 +17,9 @@ import unittest
 from typing import Callable, Optional
 
 from transformers import BatchEncoding, BertTokenizer, BertTokenizerFast, PreTrainedTokenizer, TensorType
+from transformers.testing_utils import require_tf, require_torch, slow
 from transformers.tokenization_gpt2 import GPT2Tokenizer
 
-from .utils import require_tf, require_torch, slow
-
 
 class TokenizerUtilsTest(unittest.TestCase):
     def check_tokenizer_from_pretrained(self, tokenizer_class):
diff --git a/tests/test_tokenization_xlm.py b/tests/test_tokenization_xlm.py
index 92f4c9f871..8e9d8946f2 100644
--- a/tests/test_tokenization_xlm.py
+++ b/tests/test_tokenization_xlm.py
@@ -18,10 +18,10 @@ import json
 import os
 import unittest
 
+from transformers.testing_utils import slow
 from transformers.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import slow
 
 
 class XLMTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
diff --git a/tests/test_tokenization_xlm_roberta.py b/tests/test_tokenization_xlm_roberta.py
index 57212571e6..c67e9e2f24 100644
--- a/tests/test_tokenization_xlm_roberta.py
+++ b/tests/test_tokenization_xlm_roberta.py
@@ -18,10 +18,10 @@ import os
 import unittest
 
 from transformers.file_utils import cached_property
+from transformers.testing_utils import slow
 from transformers.tokenization_xlm_roberta import SPIECE_UNDERLINE, XLMRobertaTokenizer
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import slow
 
 
 SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
diff --git a/tests/test_tokenization_xlnet.py b/tests/test_tokenization_xlnet.py
index a56e236a1b..9f92d0a05b 100644
--- a/tests/test_tokenization_xlnet.py
+++ b/tests/test_tokenization_xlnet.py
@@ -17,10 +17,10 @@
 import os
 import unittest
 
+from transformers.testing_utils import slow
 from transformers.tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer
 
 from .test_tokenization_common import TokenizerTesterMixin
-from .utils import slow
 
 
 SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
diff --git a/tests/test_trainer.py b/tests/test_trainer.py
index 7aead04429..141c7128d2 100644
--- a/tests/test_trainer.py
+++ b/tests/test_trainer.py
@@ -1,8 +1,7 @@
 import unittest
 
 from transformers import AutoTokenizer, TrainingArguments, is_torch_available
-
-from .utils import require_torch
+from transformers.testing_utils import require_torch
 
 
 if is_torch_available():