refactor: Make direct_transformers_import util (#21652)

* refactor: Make direct_import util

* edit direct import fn

* add docstring

* make import function specific to transformers only

* edit doc string
This commit is contained in:
Connor Henderson
2023-02-16 11:32:32 -05:00
committed by GitHub
parent 96d4fa46ed
commit 0f96c26de6
12 changed files with 47 additions and 101 deletions

View File

@@ -15,10 +15,10 @@
import argparse
import collections
import importlib.util
import os
import re
import sys
from transformers.utils import direct_transformers_import
# All paths are set with the intent you should run this script from the root of the repo with the command
@@ -64,14 +64,7 @@ _re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGenerati
# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
"transformers",
os.path.join(TRANSFORMERS_PATH, "__init__.py"),
submodule_search_locations=[TRANSFORMERS_PATH],
)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
transformers_module = sys.modules["transformers"]
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
# Thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python