Replace pkg_resources with importlib_metadata (#11061)

* Replace pkg_resources with importlib_metadata

Fixes #10964. The other reason for this change is that pkg_resources has been [deprecated](8fe85c22ce) in favor of importlib_metadata.

* Reduce to a single importlib_metadata import switch

* Trigger CI

Co-authored-by: Stas Bekman <stas@stason.org>

src/transformers/file_utils.py

@@ -46,19 +46,13 @@ from tqdm.auto import tqdm
 
 import requests
 from filelock import FileLock
+from transformers.utils.versions import importlib_metadata
 
 from . import __version__
 from .hf_api import HfFolder
 from .utils import logging
 
 
-# The package importlib_metadata is in a different place, depending on the python version.
-if sys.version_info < (3, 8):
-    import importlib_metadata
-else:
-    import importlib.metadata as importlib_metadata
-
-
 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
 
 ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}

src/transformers/utils/versions.py

@@ -22,7 +22,12 @@ from typing import Optional
 
 from packaging import version
 
-import pkg_resources
+
+# The package importlib_metadata is in a different place, depending on the python version.
+if sys.version_info < (3, 8):
+    import importlib_metadata
+else:
+    import importlib.metadata as importlib_metadata
 
 
 ops = {
@@ -39,7 +44,7 @@ def require_version(requirement: str, hint: Optional[str] = None) -> None:
     """
     Perform a runtime check of the dependency versions, using the exact same syntax used by pip.
 
-    The installed module version comes from the `site-packages` dir via `pkg_resources`.
+    The installed module version comes from the `site-packages` dir via `importlib_metadata`.
 
     Args:
         requirement (:obj:`str`): pip style definition, e.g.,  "tokenizers==0.9.4", "tqdm>=4.27", "numpy"
@@ -70,20 +75,22 @@ def require_version(requirement: str, hint: Optional[str] = None) -> None:
     if pkg == "python":
         got_ver = ".".join([str(x) for x in sys.version_info[:3]])
         if not ops[op](version.parse(got_ver), version.parse(want_ver)):
-            raise pkg_resources.VersionConflict(
+            raise ImportError(
                 f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}."
             )
         return
 
     # check if any version is installed
     try:
-        got_ver = pkg_resources.get_distribution(pkg).version
+        got_ver = importlib_metadata.version(pkg)
-    except pkg_resources.DistributionNotFound:
+    except importlib_metadata.PackageNotFoundError:
-        raise pkg_resources.DistributionNotFound(requirement, ["this application", hint])
+        raise importlib_metadata.PackageNotFoundError(
+            f"The '{requirement}' distribution was not found and is required by this application. {hint}"
+        )
 
     # check that the right version is installed if version number was provided
     if want_ver is not None and not ops[op](version.parse(got_ver), version.parse(want_ver)):
-        raise pkg_resources.VersionConflict(
+        raise ImportError(
             f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}"
         )
 

tests/test_versions_utils.py

@@ -16,9 +16,13 @@ import sys
 
 import numpy
 
-import pkg_resources
 from transformers.testing_utils import TestCasePlus
-from transformers.utils.versions import require_version, require_version_core, require_version_examples
+from transformers.utils.versions import (
+    importlib_metadata,
+    require_version,
+    require_version_core,
+    require_version_examples,
+)
 
 
 numpy_ver = numpy.__version__
@@ -57,7 +61,7 @@ class DependencyVersionCheckTest(TestCasePlus):
         for req in ["numpy==1.0.0", "numpy>=1000.0.0", f"numpy<{numpy_ver}"]:
             try:
                 require_version_core(req)
-            except pkg_resources.VersionConflict as e:
+            except ImportError as e:
                 self.assertIn(f"{req} is required", str(e))
                 self.assertIn("but found", str(e))
 
@@ -65,7 +69,7 @@ class DependencyVersionCheckTest(TestCasePlus):
         for req in ["numpipypie>1", "numpipypie2"]:
             try:
                 require_version_core(req)
-            except pkg_resources.DistributionNotFound as e:
+            except importlib_metadata.PackageNotFoundError as e:
                 self.assertIn(f"The '{req}' distribution was not found and is required by this application", str(e))
                 self.assertIn("Try: pip install transformers -U", str(e))
 
@@ -87,7 +91,7 @@ class DependencyVersionCheckTest(TestCasePlus):
         # the main functionality is tested in `test_core`, this is just the hint check
         try:
             require_version_examples("numpy>1000.4.5")
-        except pkg_resources.VersionConflict as e:
+        except ImportError as e:
             self.assertIn("is required", str(e))
             self.assertIn("pip install -r examples/requirements.txt", str(e))
 
@@ -100,6 +104,6 @@ class DependencyVersionCheckTest(TestCasePlus):
         for req in ["python>9.9.9", "python<3.0.0"]:
             try:
                 require_version_core(req)
-            except pkg_resources.VersionConflict as e:
+            except ImportError as e:
                 self.assertIn(f"{req} is required", str(e))
                 self.assertIn(f"but found python=={python_ver}", str(e))