diff --git a/setup.py b/setup.py
index eb954ad71d..80b4941ea5 100644
--- a/setup.py
+++ b/setup.py
@@ -102,7 +102,7 @@ if stale_egg_info.exists():
 # 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
 _deps = [
     "Pillow",
-    "accelerate>=0.10.0",
+    "accelerate>=0.17.0",
     "av==9.2.0",  # Latest version of PyAV (10.0.0) has issues with audio stream.
     "beautifulsoup4",
     "black~=23.1",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index 9c8b14711e..9206683c49 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -3,7 +3,7 @@
 # 2. run `make deps_table_update``
 deps = {
     "Pillow": "Pillow",
-    "accelerate": "accelerate>=0.10.0",
+    "accelerate": "accelerate>=0.17.0",
     "av": "av==9.2.0",
     "beautifulsoup4": "beautifulsoup4",
     "black": "black~=23.1",
diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py
index 6af20f4549..e1aee1b8d5 100644
--- a/src/transformers/training_args.py
+++ b/src/transformers/training_args.py
@@ -1552,10 +1552,10 @@ class TrainingArguments:
         if (
             torch.distributed.is_available()
             and torch.distributed.is_initialized()
-            and self.distributed_state.distributed_type != DistributedType.NO
+            and self.distributed_state.distributed_type == DistributedType.NO
         ):
             logger.warning(
-                "torch.distributed process group is initialized, but parallel_mode == ParallelMode.DISTRIBUTED. "
+                "torch.distributed process group is initialized, but parallel_mode != ParallelMode.DISTRIBUTED. "
                 "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"
             )