SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add possibility to ignore imports in test_fecther (#12801)

Browse Source 
* Add possibility to ignore imports in test_fecther

* Style
This commit is contained in:
Sylvain Gugger
2021-07-26 09:48:19 -04:00
committed by GitHub
parent 7c300d6d42
commit 5f43623843
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/tokenization_utils_base.py
View File

@@ -1769,7 +1769,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
init_kwargs = init_configuration init_kwargs = init_configuration
if config_tokenizer_class is None: if config_tokenizer_class is None:
from .models.auto.configuration_auto import AutoConfig from .models.auto.configuration_auto import AutoConfig # tests_ignore
# Second attempt. If we have not yet found tokenizer_class, let's try to use the config. # Second attempt. If we have not yet found tokenizer_class, let's try to use the config.
try: try:
@@ -1781,8 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
if config_tokenizer_class is None: if config_tokenizer_class is None:
# Third attempt. If we have not yet found the original type of the tokenizer, # Third attempt. If we have not yet found the original type of the tokenizer,
# we are loading we see if we can infer it from the type of the configuration file # we are loading we see if we can infer it from the type of the configuration file
from .models.auto.configuration_auto import CONFIG_MAPPING from .models.auto.configuration_auto import CONFIG_MAPPING # tests_ignore
from .models.auto.tokenization_auto import TOKENIZER_MAPPING from .models.auto.tokenization_auto import TOKENIZER_MAPPING # tests_ignore
if hasattr(config, "model_type"): if hasattr(config, "model_type"):
config_class = CONFIG_MAPPING.get(config.model_type) config_class = CONFIG_MAPPING.get(config.model_type)

6
utils/tests_fetcher.py
View File

@@ -131,7 +131,8 @@ def get_module_dependencies(module_fname):
imported_modules = [] imported_modules = []
# Let's start with relative imports # Let's start with relative imports
relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+\S+\s", content) relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+([^\n]+)\n", content)
relative_imports = [mod for mod, imp in relative_imports if "# tests_ignore" not in imp]
for imp in relative_imports: for imp in relative_imports:
level = 0 level = 0
while imp.startswith("."): while imp.startswith("."):
@@ -151,7 +152,8 @@ def get_module_dependencies(module_fname):
# Let's continue with direct imports # Let's continue with direct imports
# The import from the transformers module are ignored for the same reason we ignored the # The import from the transformers module are ignored for the same reason we ignored the
# main init before. # main init before.
direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+\S+\s", content) direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+([^\n]+)\n", content)
direct_imports = [mod for mod, imp in direct_imports if "# tests_ignore" not in imp]
for imp in direct_imports: for imp in direct_imports:
import_parts = imp.split(".") import_parts = imp.split(".")
dep_parts = ["src", "transformers"] + import_parts dep_parts = ["src", "transformers"] + import_parts