SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix check repo utils (#8600)

Browse Source
This commit is contained in:
Sylvain Gugger
2020-11-17 14:01:46 -05:00
committed by GitHub
parent f0435f5a61
commit 7f3b41a306
1 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
utils/check_repo.py
View File

@@ -49,6 +49,7 @@ TEST_FILES_WITH_NO_COMMON_TESTS = [
"test_modeling_mt5.py", "test_modeling_mt5.py",
"test_modeling_pegasus.py", "test_modeling_pegasus.py",
"test_modeling_tf_camembert.py", "test_modeling_tf_camembert.py",
"test_modeling_tf_mt5.py",
"test_modeling_tf_xlm_roberta.py", "test_modeling_tf_xlm_roberta.py",
"test_modeling_xlm_prophetnet.py", "test_modeling_xlm_prophetnet.py",
"test_modeling_xlm_roberta.py", "test_modeling_xlm_roberta.py",
@@ -62,7 +63,6 @@ IGNORE_NON_DOCUMENTED = [
"T5Stack", # Building part of bigger (tested) model. "T5Stack", # Building part of bigger (tested) model.
"TFDPREncoder", # Building part of bigger (documented) model. "TFDPREncoder", # Building part of bigger (documented) model.
"TFDPRSpanPredictor", # Building part of bigger (documented) model. "TFDPRSpanPredictor", # Building part of bigger (documented) model.
"TFElectraMainLayer", # Building part of bigger (documented) model (should it be a TFPreTrainedModel ?)
] ]
# Update this dict with any special correspondance model name (used in modeling_xxx.py) to doc file. # Update this dict with any special correspondance model name (used in modeling_xxx.py) to doc file.
@@ -135,11 +135,15 @@ def get_model_modules():
"modeling_tf_transfo_xl_utilities", "modeling_tf_transfo_xl_utilities",
] ]
modules = [] modules = []
for attr_name in dir(transformers): for model in dir(transformers.models):
if attr_name.startswith("modeling") and attr_name not in _ignore_modules: # There are some magic dunder attributes in the dir, we ignore them
module = getattr(transformers, attr_name) if not model.startswith("__"):
if inspect.ismodule(module): model_module = getattr(transformers.models, model)
modules.append(module) for submodule in dir(model_module):
if submodule.startswith("modeling") and submodule not in _ignore_modules:
modeling_module = getattr(model_module, submodule)
if inspect.ismodule(modeling_module):
modules.append(modeling_module)
return modules return modules
@@ -244,7 +248,7 @@ def check_all_models_are_tested():
test_files = get_model_test_files() test_files = get_model_test_files()
failures = [] failures = []
for module in modules: for module in modules:
test_file = f"test_{module.__name__.split('.')[1]}.py" test_file = f"test_{module.__name__.split('.')[-1]}.py"
if test_file not in test_files: if test_file not in test_files:
failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.") failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
new_failures = check_models_are_tested(module, test_file) new_failures = check_models_are_tested(module, test_file)
@@ -279,9 +283,9 @@ def check_models_are_documented(module, doc_file):
def _get_model_name(module): def _get_model_name(module):
""" Get the model name for the module defining it.""" """ Get the model name for the module defining it."""
splits = module.__name__.split("_") module_name = module.__name__.split(".")[-1]
splits = module_name.split("_")
splits = splits[(2 if splits[1] in ["flax", "tf"] else 1) :] splits = splits[(2 if splits[1] in ["flax", "tf"] else 1) :]
return "_".join(splits) return "_".join(splits)