diff --git a/docs/source/en/testing.mdx b/docs/source/en/testing.mdx
index cb03a57b04..4663b8ac4d 100644
--- a/docs/source/en/testing.mdx
+++ b/docs/source/en/testing.mdx
@@ -176,6 +176,15 @@ If you want to include only tests that include both patterns, `and` is to be use
 ```bash
 pytest -k "test and ada" tests/test_optimization.py
 ```
+
+### Run `accelerate` tests
+
+Sometimes you need to run `accelerate` tests on your models. For that you can just add `-m accelerate_tests` to your command, if let's say you want to run these tests on `OPT` run:
+```bash
+RUN_SLOW=1 pytest -m accelerate_tests tests/models/opt/test_modeling_opt.py 
+```
+
+
 ### Run documentation tests 
 
 In order to test whether the documentation examples are correct, you should check that the `doctests` are passing. 
diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py
index ed614abbc5..816a959a4e 100755
--- a/tests/test_modeling_common.py
+++ b/tests/test_modeling_common.py
@@ -32,6 +32,7 @@ from typing import Dict, List, Tuple
 import numpy as np
 from huggingface_hub import HfFolder, delete_repo, set_access_token
 from huggingface_hub.file_download import http_get
+from pytest import mark
 from requests.exceptions import HTTPError
 
 import transformers
@@ -2463,6 +2464,7 @@ class ModelTesterMixin:
                 self.assertEqual(param.device, torch.device(param_device))
 
     @require_accelerate
+    @mark.accelerate_tests
     @require_torch_gpu
     def test_disk_offload(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
@@ -2498,6 +2500,7 @@ class ModelTesterMixin:
                 self.assertTrue(torch.allclose(base_output[0], new_output[0]))
 
     @require_accelerate
+    @mark.accelerate_tests
     @require_torch_gpu
     def test_cpu_offload(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
@@ -2533,6 +2536,7 @@ class ModelTesterMixin:
                     self.assertTrue(torch.allclose(base_output[0], new_output[0]))
 
     @require_accelerate
+    @mark.accelerate_tests
     @require_torch_multi_gpu
     def test_model_parallelism(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
@@ -3172,6 +3176,7 @@ class ModelUtilsTest(TestCasePlus):
         self.assertIsNotNone(model)
 
     @require_accelerate
+    @mark.accelerate_tests
     def test_from_pretrained_low_cpu_mem_usage_functional(self):
         # test that we can use `from_pretrained(..., low_cpu_mem_usage=True)` with normal and
         # sharded models
@@ -3185,6 +3190,7 @@ class ModelUtilsTest(TestCasePlus):
 
     @require_usr_bin_time
     @require_accelerate
+    @mark.accelerate_tests
     def test_from_pretrained_low_cpu_mem_usage_measured(self):
         # test that `from_pretrained(..., low_cpu_mem_usage=True)` uses less cpu memory than default
 
@@ -3224,6 +3230,7 @@ class ModelUtilsTest(TestCasePlus):
         # cuda memory tracking and then we should be able to do a much more precise test.
 
     @require_accelerate
+    @mark.accelerate_tests
     @require_torch_multi_gpu
     @slow
     def test_model_parallelism_gpt2(self):
@@ -3241,6 +3248,7 @@ class ModelUtilsTest(TestCasePlus):
         self.assertEqual(text_output, "Hello, my name is John. I'm a writer, and I'm a writer. I'm")
 
     @require_accelerate
+    @mark.accelerate_tests
     @require_torch_gpu
     def test_from_pretrained_disk_offload_task_model(self):
         model = AutoModel.from_pretrained("hf-internal-testing/tiny-random-gpt2")