SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

refactor: Make direct_transformers_import util (#21652)

Browse Source 
* refactor: Make direct_import util

* edit direct import fn

* add docstring

* make import function specific to transformers only

* edit doc string
This commit is contained in:
Connor Henderson
2023-02-16 11:32:32 -05:00
committed by GitHub
parent 96d4fa46ed
commit 0f96c26de6
12 changed files with 47 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/transformers/processing_utils.py
View File

@@ -16,25 +16,18 @@
Processing saving/loading class for common processors. Processing saving/loading class for common processors.
""" """
import importlib.util
import os import os
import sys
from pathlib import Path from pathlib import Path
from .dynamic_module_utils import custom_object_save from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, logging from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging
logger = logging.get_logger(__name__) logger = logging.get_logger(__name__)
# Dynamically import the Transformers module to grab the attribute classes of the processor form their names. # Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
spec = importlib.util.spec_from_file_location( transformers_module = direct_transformers_import(Path(__file__).parent)
"transformers", Path(__file__).parent / "__init__.py", submodule_search_locations=[Path(__file__).parent]
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
AUTO_TO_BASE_CLASS_MAPPING = { AUTO_TO_BASE_CLASS_MAPPING = {

1
src/transformers/utils/__init__.py
View File

@@ -94,6 +94,7 @@ from .import_utils import (
OptionalDependencyNotAvailable, OptionalDependencyNotAvailable,
_LazyModule, _LazyModule,
ccl_version, ccl_version,
direct_transformers_import,
is_accelerate_available, is_accelerate_available,
is_apex_available, is_apex_available,
is_bitsandbytes_available, is_bitsandbytes_available,

19
src/transformers/utils/import_utils.py
View File

@@ -1129,3 +1129,22 @@ class _LazyModule(ModuleType):
class OptionalDependencyNotAvailable(BaseException): class OptionalDependencyNotAvailable(BaseException):
"""Internally used error class for signalling an optional dependency was not found.""" """Internally used error class for signalling an optional dependency was not found."""
def direct_transformers_import(path: str, file="__init__.py") -> ModuleType:
"""Imports transformers directly
Args:
path (`str`): The path to the source file
file (`str`, optional): The file to join with the path. Defaults to "__init__.py".
Returns:
`ModuleType`: The resulting imported module
"""
name = "transformers"
location = os.path.join(path, file)
spec = importlib.util.spec_from_file_location(name, location, submodule_search_locations=[path])
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module = sys.modules[name]
return module

12
tests/pipelines/test_pipelines_common.py
View File

@@ -13,7 +13,6 @@
# limitations under the License. # limitations under the License.
import copy import copy
import importlib
import logging import logging
import os import os
import random import random
@@ -53,7 +52,7 @@ from transformers.testing_utils import (
require_torch_or_tf, require_torch_or_tf,
slow, slow,
) )
from transformers.utils import is_tf_available, is_torch_available from transformers.utils import direct_transformers_import, is_tf_available, is_torch_available
from transformers.utils import logging as transformers_logging from transformers.utils import logging as transformers_logging
@@ -69,14 +68,7 @@ PATH_TO_TRANSFORMERS = os.path.join(Path(__file__).parent.parent.parent, "src/tr
# Dynamically import the Transformers module to grab the attribute classes of the processor form their names. # Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
spec = importlib.util.spec_from_file_location( transformers_module = direct_transformers_import(PATH_TO_TRANSFORMERS)
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
class ANY: class ANY:

13
utils/check_config_attributes.py
View File

@@ -13,11 +13,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import importlib
import inspect import inspect
import os import os
import re import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
@@ -26,14 +26,7 @@ PATH_TO_TRANSFORMERS = "src/transformers"
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING

14
utils/check_config_docstrings.py
View File

@@ -13,11 +13,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import importlib
import inspect import inspect
import os
import re import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
@@ -26,14 +25,7 @@ PATH_TO_TRANSFORMERS = "src/transformers"
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING

13
utils/check_copies.py
View File

@@ -15,14 +15,14 @@
import argparse import argparse
import glob import glob
import importlib.util
import os import os
import re import re
import sys
import black import black
from doc_builder.style_doc import style_docstrings_in_code from doc_builder.style_doc import style_docstrings_in_code
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py # python utils/check_copies.py
@@ -99,14 +99,7 @@ LOCALIZED_READMES = {
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
def _should_continue(line, indent): def _should_continue(line, indent):

13
utils/check_inits.py
View File

@@ -14,10 +14,8 @@
# limitations under the License. # limitations under the License.
import collections import collections
import importlib.util
import os import os
import re import re
import sys
from pathlib import Path from pathlib import Path
@@ -275,14 +273,9 @@ IGNORE_SUBMODULES = [
def check_submodules(): def check_submodules():
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( from transformers.utils import direct_transformers_import
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
module_not_registered = [ module_not_registered = [
module module

13
utils/check_repo.py
View File

@@ -13,11 +13,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import importlib
import inspect import inspect
import os import os
import re import re
import sys
import warnings import warnings
from collections import OrderedDict from collections import OrderedDict
from difflib import get_close_matches from difflib import get_close_matches
@@ -25,7 +23,7 @@ from pathlib import Path
from transformers import is_flax_available, is_tf_available, is_torch_available from transformers import is_flax_available, is_tf_available, is_torch_available
from transformers.models.auto import get_values from transformers.models.auto import get_values
from transformers.utils import ENV_VARS_TRUE_VALUES from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
@@ -307,14 +305,7 @@ MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
"transformers",
os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers)
transformers = sys.modules["transformers"]
def check_model_list(): def check_model_list():

13
utils/check_table.py
View File

@@ -15,10 +15,10 @@
import argparse import argparse
import collections import collections
import importlib.util
import os import os
import re import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
@@ -64,14 +64,7 @@ _re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGenerati
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
# Thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python # Thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python

13
utils/check_task_guides.py
View File

@@ -14,9 +14,9 @@
# limitations under the License. # limitations under the License.
import argparse import argparse
import importlib.util
import os import os
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
@@ -52,14 +52,7 @@ def _find_text_in_file(filename, start_prompt, end_prompt):
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
TASK_GUIDE_TO_MODELS = { TASK_GUIDE_TO_MODELS = {
"asr.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES, "asr.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,

13
utils/update_metadata.py
View File

@@ -15,16 +15,16 @@
import argparse import argparse
import collections import collections
import importlib.util
import os import os
import re import re
import sys
import tempfile import tempfile
import pandas as pd import pandas as pd
from datasets import Dataset from datasets import Dataset
from huggingface_hub import Repository from huggingface_hub import Repository
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command # All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py # python utils/update_metadata.py
@@ -32,14 +32,7 @@ TRANSFORMERS_PATH = "src/transformers"
# This is to make sure the transformers module imported is the one in the repo. # This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location( transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
# Regexes that match TF/Flax/PT model names. # Regexes that match TF/Flax/PT model names.