🚨🚨🚨 Delete conversion scripts when making release wheels (#35296)

* Delete conversion scripts when making release wheels

* make fixup

* Update docstring
This commit is contained in:
Matt
2024-12-17 14:18:42 +00:00
committed by GitHub
parent 6eb00dd2f0
commit e0ae9b5974
2 changed files with 19 additions and 4 deletions

View File

@@ -20,7 +20,7 @@ To create the package for pypi.
1. Create the release branch named: v<RELEASE>-release, for example v4.19-release. For a patch release checkout the 1. Create the release branch named: v<RELEASE>-release, for example v4.19-release. For a patch release checkout the
current release branch. current release branch.
If releasing on a special branch, copy the updated README.md on the main branch for your the commit you will make If releasing on a special branch, copy the updated README.md on the main branch for the commit you will make
for the post-release and run `make fix-copies` on the main branch as well. for the post-release and run `make fix-copies` on the main branch as well.
2. Run `make pre-release` (or `make pre-patch` for a patch release) and commit these changes with the message: 2. Run `make pre-release` (or `make pre-patch` for a patch release) and commit these changes with the message:

View File

@@ -45,12 +45,14 @@ or use `make post-release`.
import argparse import argparse
import os import os
import re import re
from pathlib import Path
import packaging.version import packaging.version
# All paths are defined with the intent that this script should be run from the root of the repo. # All paths are defined with the intent that this script should be run from the root of the repo.
PATH_TO_EXAMPLES = "examples/" PATH_TO_EXAMPLES = "examples/"
PATH_TO_MODELS = "src/transformers/models"
# This maps a type of file to the pattern to look for when searching where the version is defined, as well as the # This maps a type of file to the pattern to look for when searching where the version is defined, as well as the
# template to follow when replacing it with the new version. # template to follow when replacing it with the new version.
REPLACE_PATTERNS = { REPLACE_PATTERNS = {
@@ -117,6 +119,17 @@ def global_version_update(version: str, patch: bool = False):
update_version_in_examples(version) update_version_in_examples(version)
def remove_conversion_scripts():
"""
Delete the scripts that convert models from older, unsupported formats. We don't want to include these
in release wheels because they often have to open insecure file types (pickle, Torch .bin models). This results in
vulnerability scanners flagging us and can cause compliance issues for users with strict security policies.
"""
model_dir = Path(PATH_TO_MODELS)
for conversion_script in list(model_dir.glob("**/convert*.py")):
conversion_script.unlink()
def get_version() -> packaging.version.Version: def get_version() -> packaging.version.Version:
""" """
Reads the current version in the main __init__. Reads the current version in the main __init__.
@@ -131,7 +144,7 @@ def pre_release_work(patch: bool = False):
""" """
Do all the necessary pre-release steps: Do all the necessary pre-release steps:
- figure out the next minor release version and ask confirmation - figure out the next minor release version and ask confirmation
- update the version eveywhere - update the version everywhere
- clean-up the model list in the main README - clean-up the model list in the main README
Args: Args:
@@ -155,13 +168,15 @@ def pre_release_work(patch: bool = False):
print(f"Updating version to {version}.") print(f"Updating version to {version}.")
global_version_update(version, patch=patch) global_version_update(version, patch=patch)
print("Deleting conversion scripts.")
remove_conversion_scripts()
def post_release_work(): def post_release_work():
""" """
Do all the necesarry post-release steps: Do all the necessary post-release steps:
- figure out the next dev version and ask confirmation - figure out the next dev version and ask confirmation
- update the version eveywhere - update the version everywhere
- clean-up the model list in the main README - clean-up the model list in the main README
""" """
# First let's get the current version # First let's get the current version