SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix and refactor check_repo (#11127)

Browse Source
This commit is contained in:
Sylvain Gugger
2021-04-07 17:56:21 -04:00
committed by GitHub
parent 3fd7eee18f
commit ffe0761777
1 changed files with 13 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
utils/check_repo.py
View File

@@ -79,60 +79,26 @@ TEST_FILES_WITH_NO_COMMON_TESTS = [
# should **not** be the rule. # should **not** be the rule.
IGNORE_NON_AUTO_CONFIGURED = [ IGNORE_NON_AUTO_CONFIGURED = [
# models to ignore for model xxx mapping # models to ignore for model xxx mapping
"M2M100Encoder",
"M2M100Decoder",
"Speech2TextEncoder",
"Speech2TextDecoder",
"LEDEncoder",
"LEDDecoder",
"BartDecoder",
"BartDecoderWrapper",
"BartEncoder",
"BlenderbotSmallEncoder",
"BlenderbotSmallDecoder",
"BlenderbotSmallDecoderWrapper",
"BlenderbotEncoder",
"BlenderbotDecoder",
"BlenderbotDecoderWrapper",
"DPRContextEncoder",
"DPREncoder",
"DPRReader", "DPRReader",
"DPRSpanPredictor", "DPRSpanPredictor",
"FlaubertForQuestionAnswering", "FlaubertForQuestionAnswering",
"FunnelBaseModel", "FunnelBaseModel",
"GPT2DoubleHeadsModel", "GPT2DoubleHeadsModel",
"MT5EncoderModel",
"MBartEncoder",
"MBartDecoder",
"MBartDecoderWrapper",
"OpenAIGPTDoubleHeadsModel", "OpenAIGPTDoubleHeadsModel",
"PegasusEncoder",
"PegasusDecoder",
"PegasusDecoderWrapper",
"ProphetNetDecoder",
"ProphetNetEncoder",
"ProphetNetDecoderWrapper",
"RagModel", "RagModel",
"RagSequenceForGeneration", "RagSequenceForGeneration",
"RagTokenForGeneration", "RagTokenForGeneration",
"T5Stack", "T5Stack",
"T5EncoderModel",
"TFDPRContextEncoder",
"TFDPREncoder",
"TFDPRReader", "TFDPRReader",
"TFDPRSpanPredictor", "TFDPRSpanPredictor",
"TFFunnelBaseModel", "TFFunnelBaseModel",
"TFGPT2DoubleHeadsModel", "TFGPT2DoubleHeadsModel",
"TFMT5EncoderModel",
"TFOpenAIGPTDoubleHeadsModel", "TFOpenAIGPTDoubleHeadsModel",
"TFRagModel", "TFRagModel",
"TFRagSequenceForGeneration", "TFRagSequenceForGeneration",
"TFRagTokenForGeneration", "TFRagTokenForGeneration",
"TFT5EncoderModel",
"Wav2Vec2ForCTC", "Wav2Vec2ForCTC",
"XLMForQuestionAnswering", "XLMForQuestionAnswering",
"XLMProphetNetDecoder",
"XLMProphetNetEncoder",
"XLNetForQuestionAnswering", "XLNetForQuestionAnswering",
"SeparableConv1D", "SeparableConv1D",
] ]
@@ -286,12 +252,23 @@ def get_all_auto_configured_models():
return [cls.__name__ for cls in result] return [cls.__name__ for cls in result]
def ignore_unautoclassed(model_name):
"""Rules to determine if `name` should be in an auto class."""
# Special white list
if model_name in IGNORE_NON_AUTO_CONFIGURED:
return True
# Encoder and Decoder should be ignored
if "Encoder" in model_name or "Decoder" in model_name:
return True
return False
def check_models_are_auto_configured(module, all_auto_models): def check_models_are_auto_configured(module, all_auto_models):
""" Check models defined in module are each in an auto class.""" """ Check models defined in module are each in an auto class."""
defined_models = get_models(module) defined_models = get_models(module)
failures = [] failures = []
for model_name, _ in defined_models: for model_name, _ in defined_models:
if model_name not in all_auto_models and model_name not in IGNORE_NON_AUTO_CONFIGURED: if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
failures.append( failures.append(
f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. " f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
"If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file " "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
@@ -414,6 +391,7 @@ UNDOCUMENTED_OBJECTS = [
"convert_tf_weight_name_to_pt_weight_name", # Internal used to convert model weights "convert_tf_weight_name_to_pt_weight_name", # Internal used to convert model weights
"logger", # Internal logger "logger", # Internal logger
"logging", # External module "logging", # External module
"requires_backends", # Internal function
] ]
# This list should be empty. Objects in it should get their own doc page. # This list should be empty. Objects in it should get their own doc page.