SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix modular edge case + modular sorting order (#35562)

Browse Source 
* look-ahead negation

* re add examples by default

* Fix the bug in topological sort

* Update create_dependency_mapping.py

* start adding test

* finalize test

* more tests

* style

* style
This commit is contained in:
Cyril Vallez
2025-01-09 17:17:52 +01:00
committed by GitHub
parent d3fe9fa3fe
commit 46276f9a7f
9 changed files with 98 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
utils/modular_model_converter.py
View File

@@ -58,7 +58,7 @@ def get_module_source_from_name(module_name: str) -> str:
def preserve_case_replace(text, patterns: dict, default_name: str):
# Create a regex pattern to match all variations
regex_pattern = "|".join(re.escape(key) for key in patterns.keys())
compiled_regex = re.compile(f"({regex_pattern})(.|$)", re.IGNORECASE | re.DOTALL)
compiled_regex = re.compile(f"(?<![a-z0-9])({regex_pattern})(.|$)", re.IGNORECASE | re.DOTALL)
def replace(match):
matched_pattern = match.group(1)
@@ -1691,9 +1691,13 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.files_to_parse == ["all"]:
args.files_to_parse = glob.glob("src/transformers/models/**/modular_*.py", recursive=True)
args.files_to_parse += glob.glob("examples/**/modular_*.py", recursive=True)
if args.files_to_parse == ["examples"]:
args.files_to_parse = glob.glob("examples/**/modular_*.py", recursive=True)
for file_name in find_priority_list(args.files_to_parse):
priority_list = find_priority_list(args.files_to_parse)
assert len(priority_list) == len(args.files_to_parse), "Some files will not be converted"
for file_name in priority_list:
print(f"Converting {file_name} to a single model single file format")
module_path = file_name.replace("/", ".").replace(".py", "").replace("src.", "")
converted_files = convert_modular_file(file_name)