diff --git a/setup.py b/setup.py
index ce609f2404..b24a572073 100644
--- a/setup.py
+++ b/setup.py
@@ -76,10 +76,9 @@ To create the package for pypi.
 import os
 import re
 import shutil
-from distutils.core import Command
 from pathlib import Path
 
-from setuptools import find_packages, setup
+from setuptools import find_packages, setup, Command
 
 
 # Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index 3fd334d71c..8d958a8250 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -28,7 +28,6 @@ import tempfile
 import time
 import unittest
 from collections.abc import Mapping
-from distutils.util import strtobool
 from io import StringIO
 from pathlib import Path
 from typing import Iterator, List, Optional, Union
@@ -93,6 +92,7 @@ from .utils import (
     is_torchdynamo_available,
     is_torchvision_available,
     is_vision_available,
+    strtobool,
 )
 
 
diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index de04f8b674..ab79e94a68 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -29,7 +29,6 @@ import sys
 import time
 import warnings
 from collections.abc import Mapping
-from distutils.util import strtobool
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
 
@@ -152,6 +151,7 @@ from .utils import (
     is_torch_neuroncore_available,
     is_torch_tpu_available,
     logging,
+    strtobool,
 )
 from .utils.generic import ContextManagers
 
diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py
index b9b1771e68..132c707f42 100644
--- a/src/transformers/utils/__init__.py
+++ b/src/transformers/utils/__init__.py
@@ -47,6 +47,7 @@ from .generic import (
     is_torch_tensor,
     reshape,
     squeeze,
+    strtobool,
     tensor_size,
     to_numpy,
     to_py_obj,
diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py
index 3c3ade198b..65d92e1954 100644
--- a/src/transformers/utils/generic.py
+++ b/src/transformers/utils/generic.py
@@ -56,6 +56,21 @@ class cached_property(property):
         return cached
 
 
+# vendored from distutils.util
+def strtobool(val):
+    """Convert a string representation of truth to true (1) or false (0).
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are 'n', 'no', 'f', 'false', 'off', and '0'.
+    Raises ValueError if 'val' is anything else.
+    """
+    val = val.lower()
+    if val in {"y", "yes", "t", "true", "on", "1"}:
+        return 1
+    if val in {"n", "no", "f", "false", "off", "0"}:
+        return 0
+    raise ValueError(f"invalid truth value {val!r}")
+
+
 def is_tensor(x):
     """
     Tests if `x` is a `torch.Tensor`, `tf.Tensor`, `jaxlib.xla_extension.DeviceArray` or `np.ndarray`.