From 8a2f062eac8266d73a46082eb495a47261b56b17 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 12 Feb 2025 12:23:58 +0000 Subject: [PATCH] [commands] remove deprecated/inoperational commands (#35718) rm deprecated/inoperational commands --- src/transformers/commands/lfs.py | 226 ------------------ src/transformers/commands/transformers_cli.py | 4 - src/transformers/commands/user.py | 197 --------------- utils/not_doctested.txt | 2 - 4 files changed, 429 deletions(-) delete mode 100644 src/transformers/commands/lfs.py delete mode 100644 src/transformers/commands/user.py diff --git a/src/transformers/commands/lfs.py b/src/transformers/commands/lfs.py deleted file mode 100644 index 25537f0791..0000000000 --- a/src/transformers/commands/lfs.py +++ /dev/null @@ -1,226 +0,0 @@ -""" -Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. - -Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py - -Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - -To launch debugger while developing: - -``` [lfs "customtransfer.multipart"] -path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client -/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ```""" - -import json -import os -import subprocess -import sys -import warnings -from argparse import ArgumentParser -from contextlib import AbstractContextManager -from typing import Dict, List, Optional - -import requests - -from ..utils import logging -from . import BaseTransformersCLICommand - - -logger = logging.get_logger(__name__) # pylint: disable=invalid-name - - -LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload" - - -class LfsCommands(BaseTransformersCLICommand): - """ - Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload - large files >5GB 🔥. Spec for LFS custom transfer agent is: - https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - This introduces two commands to the CLI: - - 1. $ transformers-cli lfs-enable-largefiles - - This should be executed once for each model repo that contains a model file >5GB. It's documented in the error - message you get if you just try to git push a 5GB file without having enabled it before. - - 2. $ transformers-cli lfs-multipart-upload - - This command is called by lfs directly and is not meant to be called by the user. - """ - - @staticmethod - def register_subcommand(parser: ArgumentParser): - enable_parser = parser.add_parser( - "lfs-enable-largefiles", - help=( - "Deprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB." - ), - ) - enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.") - enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args)) - - upload_parser = parser.add_parser( - LFS_MULTIPART_UPLOAD_COMMAND, - help=( - "Deprecated: use `huggingface-cli` instead. " - "Command will get called by git-lfs, do not call it directly." - ), - ) - upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args)) - - -class LfsEnableCommand: - def __init__(self, args): - self.args = args - - def run(self): - warnings.warn( - "Managing repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead." - ) - local_path = os.path.abspath(self.args.path) - if not os.path.isdir(local_path): - print("This does not look like a valid git repo.") - exit(1) - subprocess.run( - "git config lfs.customtransfer.multipart.path transformers-cli".split(), check=True, cwd=local_path - ) - subprocess.run( - f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), - check=True, - cwd=local_path, - ) - print("Local repo set up for largefiles") - - -def write_msg(msg: Dict): - """Write out the message in Line delimited JSON.""" - msg = json.dumps(msg) + "\n" - sys.stdout.write(msg) - sys.stdout.flush() - - -def read_msg() -> Optional[Dict]: - """Read Line delimited JSON from stdin.""" - msg = json.loads(sys.stdin.readline().strip()) - - if "terminate" in (msg.get("type"), msg.get("event")): - # terminate message received - return None - - if msg.get("event") not in ("download", "upload"): - logger.critical("Received unexpected message") - sys.exit(1) - - return msg - - -class FileSlice(AbstractContextManager): - """ - File-like object that only reads a slice of a file - - Inspired by stackoverflow.com/a/29838711/593036 - """ - - def __init__(self, filepath: str, seek_from: int, read_limit: int): - self.filepath = filepath - self.seek_from = seek_from - self.read_limit = read_limit - self.n_seen = 0 - - def __enter__(self): - self.f = open(self.filepath, "rb") - self.f.seek(self.seek_from) - return self - - def __len__(self): - total_length = os.fstat(self.f.fileno()).st_size - return min(self.read_limit, total_length - self.seek_from) - - def read(self, n=-1): - if self.n_seen >= self.read_limit: - return b"" - remaining_amount = self.read_limit - self.n_seen - data = self.f.read(remaining_amount if n < 0 else min(n, remaining_amount)) - self.n_seen += len(data) - return data - - def __iter__(self): - yield self.read(n=4 * 1024 * 1024) - - def __exit__(self, *args): - self.f.close() - - -class LfsUploadCommand: - def __init__(self, args): - self.args = args - - def run(self): - # Immediately after invoking a custom transfer process, git-lfs - # sends initiation data to the process over stdin. - # This tells the process useful information about the configuration. - init_msg = json.loads(sys.stdin.readline().strip()) - if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): - write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) - sys.exit(1) - - # The transfer process should use the information it needs from the - # initiation structure, and also perform any one-off setup tasks it - # needs to do. It should then respond on stdout with a simple empty - # confirmation structure, as follows: - write_msg({}) - - # After the initiation exchange, git-lfs will send any number of - # transfer requests to the stdin of the transfer process, in a serial sequence. - while True: - msg = read_msg() - if msg is None: - # When all transfers have been processed, git-lfs will send - # a terminate event to the stdin of the transfer process. - # On receiving this message the transfer process should - # clean up and terminate. No response is expected. - sys.exit(0) - - oid = msg["oid"] - filepath = msg["path"] - completion_url = msg["action"]["href"] - header = msg["action"]["header"] - chunk_size = int(header.pop("chunk_size")) - presigned_urls: List[str] = list(header.values()) - - parts = [] - for i, presigned_url in enumerate(presigned_urls): - with FileSlice(filepath, seek_from=i * chunk_size, read_limit=chunk_size) as data: - r = requests.put(presigned_url, data=data) - r.raise_for_status() - parts.append( - { - "etag": r.headers.get("etag"), - "partNumber": i + 1, - } - ) - # In order to support progress reporting while data is uploading / downloading, - # the transfer process should post messages to stdout - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": (i + 1) * chunk_size, - "bytesSinceLast": chunk_size, - } - ) - # Not precise but that's ok. - - r = requests.post( - completion_url, - json={ - "oid": oid, - "parts": parts, - }, - ) - r.raise_for_status() - - write_msg({"event": "complete", "oid": oid}) diff --git a/src/transformers/commands/transformers_cli.py b/src/transformers/commands/transformers_cli.py index a441bad1c1..15291dcf89 100644 --- a/src/transformers/commands/transformers_cli.py +++ b/src/transformers/commands/transformers_cli.py @@ -21,10 +21,8 @@ from .chat import ChatCommand from .convert import ConvertCommand from .download import DownloadCommand from .env import EnvironmentCommand -from .lfs import LfsCommands from .run import RunCommand from .serving import ServeCommand -from .user import UserCommands def main(): @@ -38,9 +36,7 @@ def main(): EnvironmentCommand.register_subcommand(commands_parser) RunCommand.register_subcommand(commands_parser) ServeCommand.register_subcommand(commands_parser) - UserCommands.register_subcommand(commands_parser) AddNewModelLikeCommand.register_subcommand(commands_parser) - LfsCommands.register_subcommand(commands_parser) AddFastImageProcessorCommand.register_subcommand(commands_parser) # Let's go diff --git a/src/transformers/commands/user.py b/src/transformers/commands/user.py deleted file mode 100644 index bf4072ce04..0000000000 --- a/src/transformers/commands/user.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -from argparse import ArgumentParser -from typing import List, Union - -from huggingface_hub.hf_api import HfFolder, create_repo, whoami -from requests.exceptions import HTTPError - -from . import BaseTransformersCLICommand - - -class UserCommands(BaseTransformersCLICommand): - @staticmethod - def register_subcommand(parser: ArgumentParser): - login_parser = parser.add_parser("login", help="Log in using the same credentials as on huggingface.co") - login_parser.set_defaults(func=lambda args: LoginCommand(args)) - whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.") - whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) - logout_parser = parser.add_parser("logout", help="Log out") - logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) - - # new system: git-based repo system - repo_parser = parser.add_parser( - "repo", - help="Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.", - ) - repo_subparsers = repo_parser.add_subparsers( - help="Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands" - ) - repo_create_parser = repo_subparsers.add_parser( - "create", help="Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co" - ) - repo_create_parser.add_argument( - "name", - type=str, - help="Name for your model's repo. Will be namespaced under your username to build the model id.", - ) - repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.") - repo_create_parser.add_argument("-y", "--yes", action="store_true", help="Optional: answer Yes to the prompt") - repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) - - -class ANSI: - """ - Helper for en.wikipedia.org/wiki/ANSI_escape_code - """ - - _bold = "\u001b[1m" - _red = "\u001b[31m" - _gray = "\u001b[90m" - _reset = "\u001b[0m" - - @classmethod - def bold(cls, s): - return f"{cls._bold}{s}{cls._reset}" - - @classmethod - def red(cls, s): - return f"{cls._bold}{cls._red}{s}{cls._reset}" - - @classmethod - def gray(cls, s): - return f"{cls._gray}{s}{cls._reset}" - - -def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: - """ - Inspired by: - - - stackoverflow.com/a/8356620/593036 - - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data - """ - col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] - row_format = ("{{:{}}} " * len(headers)).format(*col_widths) - lines = [] - lines.append(row_format.format(*headers)) - lines.append(row_format.format(*["-" * w for w in col_widths])) - for row in rows: - lines.append(row_format.format(*row)) - return "\n".join(lines) - - -class BaseUserCommand: - def __init__(self, args): - self.args = args - - -class LoginCommand(BaseUserCommand): - def run(self): - print( - ANSI.red( - "ERROR! `huggingface-cli login` uses an outdated login mechanism " - "that is not compatible with the Hugging Face Hub backend anymore. " - "Please use `huggingface-cli login instead." - ) - ) - - -class WhoamiCommand(BaseUserCommand): - def run(self): - print( - ANSI.red( - "WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use " - "`huggingface-cli whoami` instead." - ) - ) - token = HfFolder.get_token() - if token is None: - print("Not logged in") - exit() - try: - user, orgs = whoami(token) - print(user) - if orgs: - print(ANSI.bold("orgs: "), ",".join(orgs)) - except HTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) - - -class LogoutCommand(BaseUserCommand): - def run(self): - print( - ANSI.red( - "ERROR! `transformers-cli logout` uses an outdated logout mechanism " - "that is not compatible with the Hugging Face Hub backend anymore. " - "Please use `huggingface-cli logout instead." - ) - ) - - -class RepoCreateCommand(BaseUserCommand): - def run(self): - print( - ANSI.red( - "WARNING! Managing repositories through transformers-cli is deprecated. " - "Please use `huggingface-cli` instead." - ) - ) - token = HfFolder.get_token() - if token is None: - print("Not logged in") - exit(1) - try: - stdout = subprocess.check_output(["git", "--version"]).decode("utf-8") - print(ANSI.gray(stdout.strip())) - except FileNotFoundError: - print("Looks like you do not have git installed, please install.") - - try: - stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8") - print(ANSI.gray(stdout.strip())) - except FileNotFoundError: - print( - ANSI.red( - "Looks like you do not have git-lfs installed, please install." - " You can install from https://git-lfs.github.com/." - " Then run `git lfs install` (you only have to do this once)." - ) - ) - print("") - - user, _ = whoami(token) - namespace = self.args.organization if self.args.organization is not None else user - full_name = f"{namespace}/{self.args.name}" - print(f"You are about to create {ANSI.bold(full_name)}") - - if not self.args.yes: - choice = input("Proceed? [Y/n] ").lower() - if not (choice == "" or choice == "y" or choice == "yes"): - print("Abort") - exit() - try: - url = create_repo(repo_id=full_name, token=token) - except HTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) - print("\nYour repo now lives at:") - print(f" {ANSI.bold(url)}") - print("\nYou can clone it locally with the command below, and commit/push as usual.") - print(f"\n git clone {url}") - print("") diff --git a/utils/not_doctested.txt b/utils/not_doctested.txt index 0a36fcbd8a..24a8a4ba7a 100644 --- a/utils/not_doctested.txt +++ b/utils/not_doctested.txt @@ -343,12 +343,10 @@ src/transformers/commands/add_new_model_like.py src/transformers/commands/convert.py src/transformers/commands/download.py src/transformers/commands/env.py -src/transformers/commands/lfs.py src/transformers/commands/run.py src/transformers/commands/serving.py src/transformers/commands/train.py src/transformers/commands/transformers_cli.py -src/transformers/commands/user.py src/transformers/configuration_utils.py src/transformers/convert_graph_to_onnx.py src/transformers/convert_pytorch_checkpoint_to_tf2.py