SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

refactor: Make direct_transformers_import util (#21652)

Browse Source 
* refactor: Make direct_import util

* edit direct import fn

* add docstring

* make import function specific to transformers only

* edit doc string
This commit is contained in:
Connor Henderson
2023-02-16 11:32:32 -05:00
committed by GitHub
parent 96d4fa46ed
commit 0f96c26de6
12 changed files with 47 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
utils/check_config_attributes.py
View File

@@ -13,11 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib
import inspect
import os
import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
@@ -26,14 +26,7 @@ PATH_TO_TRANSFORMERS = "src/transformers"
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING

14
utils/check_config_docstrings.py
View File

@@ -13,11 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib
import inspect
import os
import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
@@ -26,14 +25,7 @@ PATH_TO_TRANSFORMERS = "src/transformers"
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING

13
utils/check_copies.py
View File

@@ -15,14 +15,14 @@
import argparse
import glob
import importlib.util
import os
import re
import sys
import black
from doc_builder.style_doc import style_docstrings_in_code
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
@@ -99,14 +99,7 @@ LOCALIZED_READMES = {
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
def _should_continue(line, indent):

13
utils/check_inits.py
View File

@@ -14,10 +14,8 @@
# limitations under the License.
import collections
import importlib.util
import os
import re
import sys
from pathlib import Path
@@ -275,14 +273,9 @@ IGNORE_SUBMODULES = [
def check_submodules():
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
from transformers.utils import direct_transformers_import
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
module_not_registered = [
module

13
utils/check_repo.py
View File

@@ -13,11 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib
import inspect
import os
import re
import sys
import warnings
from collections import OrderedDict
from difflib import get_close_matches
@@ -25,7 +23,7 @@ from pathlib import Path
from transformers import is_flax_available, is_tf_available, is_torch_available
from transformers.models.auto import get_values
from transformers.utils import ENV_VARS_TRUE_VALUES
from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
@@ -307,14 +305,7 @@ MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
def check_model_list():

13
utils/check_table.py
View File

@@ -15,10 +15,10 @@
import argparse
import collections
import importlib.util
import os
import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
@@ -64,14 +64,7 @@ _re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGenerati
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
# Thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python

13
utils/check_task_guides.py
View File

@@ -14,9 +14,9 @@
# limitations under the License.
import argparse
import importlib.util
import os
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
@@ -52,14 +52,7 @@ def _find_text_in_file(filename, start_prompt, end_prompt):
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
TASK_GUIDE_TO_MODELS = {
"asr.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,

13
utils/update_metadata.py
View File

@@ -15,16 +15,16 @@
import argparse
import collections
import importlib.util
import os
import re
import sys
import tempfile
import pandas as pd
from datasets import Dataset
from huggingface_hub import Repository
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
@@ -32,14 +32,7 @@ TRANSFORMERS_PATH = "src/transformers"
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
# Regexes that match TF/Flax/PT model names.