From 5f436238435e960f4172737c55f699db711361d3 Mon Sep 17 00:00:00 2001
From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
Date: Mon, 26 Jul 2021 09:48:19 -0400
Subject: [PATCH] Add possibility to ignore imports in test_fecther (#12801)

* Add possibility to ignore imports in test_fecther

* Style
---
 src/transformers/tokenization_utils_base.py | 6 +++---
 utils/tests_fetcher.py                      | 6 ++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py
index cc1d885cc3..132a852eda 100644
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -1769,7 +1769,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
             init_kwargs = init_configuration
 
         if config_tokenizer_class is None:
-            from .models.auto.configuration_auto import AutoConfig
+            from .models.auto.configuration_auto import AutoConfig  # tests_ignore
 
             # Second attempt. If we have not yet found tokenizer_class, let's try to use the config.
             try:
@@ -1781,8 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
             if config_tokenizer_class is None:
                 # Third attempt. If we have not yet found the original type of the tokenizer,
                 # we are loading we see if we can infer it from the type of the configuration file
-                from .models.auto.configuration_auto import CONFIG_MAPPING
-                from .models.auto.tokenization_auto import TOKENIZER_MAPPING
+                from .models.auto.configuration_auto import CONFIG_MAPPING  # tests_ignore
+                from .models.auto.tokenization_auto import TOKENIZER_MAPPING  # tests_ignore
 
                 if hasattr(config, "model_type"):
                     config_class = CONFIG_MAPPING.get(config.model_type)
diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py
index ee1b6e834a..6377e58894 100644
--- a/utils/tests_fetcher.py
+++ b/utils/tests_fetcher.py
@@ -131,7 +131,8 @@ def get_module_dependencies(module_fname):
     imported_modules = []
 
     # Let's start with relative imports
-    relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+\S+\s", content)
+    relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+([^\n]+)\n", content)
+    relative_imports = [mod for mod, imp in relative_imports if "# tests_ignore" not in imp]
     for imp in relative_imports:
         level = 0
         while imp.startswith("."):
@@ -151,7 +152,8 @@ def get_module_dependencies(module_fname):
     # Let's continue with direct imports
     # The import from the transformers module are ignored for the same reason we ignored the
     # main init before.
-    direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+\S+\s", content)
+    direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+([^\n]+)\n", content)
+    direct_imports = [mod for mod, imp in direct_imports if "# tests_ignore" not in imp]
     for imp in direct_imports:
         import_parts = imp.split(".")
         dep_parts = ["src", "transformers"] + import_parts