SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix test fetcher (#36129)

Browse Source 
* fix

* fix

* update

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2025-02-12 17:35:41 +01:00
committed by GitHub
parent 1fae54c721
commit 4a5a7b991a
2 changed files with 19 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/transformers/utils/import_utils.py
View File

@@ -1927,11 +1927,16 @@ def fetch__all__(file_content):
if "__all__" not in file_content: if "__all__" not in file_content:
return [] return []
start_index = None
lines = file_content.splitlines() lines = file_content.splitlines()
for index, line in enumerate(lines): for index, line in enumerate(lines):
if line.startswith("__all__"): if line.startswith("__all__"):
start_index = index start_index = index
# There is no line starting with `__all__`
if start_index is None:
return []
lines = lines[start_index:] lines = lines[start_index:]
if not lines[0].startswith("__all__"): if not lines[0].startswith("__all__"):

14
utils/tests_fetcher.py
View File

@@ -735,11 +735,23 @@ def get_module_dependencies(module_fname: str, cache: Dict[str, List[str]] = Non
if module.endswith("__init__.py"): if module.endswith("__init__.py"):
# So we get the imports from that init then try to find where our objects come from. # So we get the imports from that init then try to find where our objects come from.
new_imported_modules = extract_imports(module, cache=cache) new_imported_modules = extract_imports(module, cache=cache)
# Add imports via `define_import_structure` after the #35167 as we remove explicit import in `__init__.py`
from transformers.utils.import_utils import define_import_structure
new_imported_modules_2 = define_import_structure(PATH_TO_REPO / module)
for mapping in new_imported_modules_2.values():
for _module, _imports in mapping.items():
_module = module.replace("__init__.py", f"{_module}.py")
new_imported_modules.append((_module, list(_imports)))
for new_module, new_imports in new_imported_modules: for new_module, new_imports in new_imported_modules:
if any(i in new_imports for i in imports): if any(i in new_imports for i in imports):
if new_module not in dependencies: if new_module not in dependencies:
new_modules.append((new_module, [i for i in new_imports if i in imports])) new_modules.append((new_module, [i for i in new_imports if i in imports]))
imports = [i for i in imports if i not in new_imports] imports = [i for i in imports if i not in new_imports]
if len(imports) > 0: if len(imports) > 0:
# If there are any objects lefts, they may be a submodule # If there are any objects lefts, they may be a submodule
path_to_module = PATH_TO_REPO / module.replace("__init__.py", "") path_to_module = PATH_TO_REPO / module.replace("__init__.py", "")
@@ -759,6 +771,7 @@ def get_module_dependencies(module_fname: str, cache: Dict[str, List[str]] = Non
dependencies.append(module) dependencies.append(module)
imported_modules = new_modules imported_modules = new_modules
return dependencies return dependencies
@@ -880,6 +893,7 @@ def create_reverse_dependency_map() -> Dict[str, List[str]]:
depending on it recursively. This way the tests impacted by a change in file A are the test files in the list depending on it recursively. This way the tests impacted by a change in file A are the test files in the list
corresponding to key A in this result. corresponding to key A in this result.
""" """
cache = {} cache = {}
# Start from the example deps init. # Start from the example deps init.
example_deps, examples = init_test_examples_dependencies() example_deps, examples = init_test_examples_dependencies()