From 6a5272b2054622e27b4843e0c9722cc716d5458d Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Tue, 21 Jun 2022 17:51:18 +0200 Subject: [PATCH] Prepare transformers for v0.8.0 huggingface-hub release (#17716) * Prepare CI for v0.8.0 * pin hfh (revert before merge) * Revert "pin hfh (revert before merge)" This reverts commit a0103140e1c77b810ffcb735192968bc03be3e1f. * Test rc3 * Test latest rc * Unpin to the RC Co-authored-by: Sylvain Gugger --- src/transformers/commands/user.py | 43 ++++-------------------- src/transformers/testing_utils.py | 6 ++-- src/transformers/utils/hub.py | 2 +- tests/models/auto/test_processor_auto.py | 14 ++++---- tests/test_configuration_common.py | 14 ++++---- tests/test_feature_extraction_common.py | 14 ++++---- tests/test_modeling_common.py | 16 +++++---- tests/test_modeling_flax_common.py | 12 ++++--- tests/test_modeling_tf_common.py | 14 ++++---- tests/test_tokenization_common.py | 14 ++++---- tests/trainer/test_trainer.py | 12 ++++--- 11 files changed, 75 insertions(+), 86 deletions(-) diff --git a/src/transformers/commands/user.py b/src/transformers/commands/user.py index f3ec7f740d..2f073235d2 100644 --- a/src/transformers/commands/user.py +++ b/src/transformers/commands/user.py @@ -14,10 +14,9 @@ import subprocess from argparse import ArgumentParser -from getpass import getpass from typing import List, Union -from huggingface_hub.hf_api import HfFolder, create_repo, login, logout, whoami +from huggingface_hub.hf_api import HfFolder, create_repo, whoami from requests.exceptions import HTTPError from . import BaseTransformersCLICommand @@ -106,33 +105,11 @@ class LoginCommand(BaseUserCommand): def run(self): print( ANSI.red( - "WARNING! `transformers-cli login` is deprecated and will be removed in v5. Please use " - "`huggingface-cli login` instead." + "ERROR! `transformers-cli login` uses an outdated login mechanism " + "that is not compatible with the Hugging Face Hub backend anymore. " + "Please use `huggingface-cli login instead." ) ) - print( # docstyle-ignore - """ - _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_| - _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| - _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_| - _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| - _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_| - - """ - ) - username = input("Username: ") - password = getpass() - try: - token = login(username, password) - except HTTPError as e: - # probably invalid credentials, display error message. - print(e) - print(ANSI.red(e.response.text)) - exit(1) - HfFolder.save_token(token) - print("Login successful") - print("Your token:", token, "\n") - print("Your token has been saved to", HfFolder.path_token) class WhoamiCommand(BaseUserCommand): @@ -162,17 +139,11 @@ class LogoutCommand(BaseUserCommand): def run(self): print( ANSI.red( - "WARNING! `transformers-cli logout` is deprecated and will be removed in v5. Please use " - "`huggingface-cli logout` instead." + "ERROR! `transformers-cli logout` uses an outdated logout mechanism " + "that is not compatible with the Hugging Face Hub backend anymore. " + "Please use `huggingface-cli logout instead." ) ) - token = HfFolder.get_token() - if token is None: - print("Not logged in") - exit() - HfFolder.delete_token() - logout(token) - print("Successfully logged out.") class RepoCreateCommand(BaseUserCommand): diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 2065da6004..b73f3c380a 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -84,8 +84,10 @@ DUMMY_DIFF_TOKENIZER_IDENTIFIER = "julien-c/dummy-diff-tokenizer" # Used to test the hub USER = "__DUMMY_TRANSFORMERS_USER__" -PASS = "__DUMMY_TRANSFORMERS_PASS__" -ENDPOINT_STAGING = "https://moon-staging.huggingface.co" +ENDPOINT_STAGING = "https://hub-ci.huggingface.co" + +# Not critical, only usable on the sandboxed CI instance. +TOKEN = "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL" def parse_flag_from_env(key, default=False): diff --git a/src/transformers/utils/hub.py b/src/transformers/utils/hub.py index a4717cf7ea..a2ec88008e 100644 --- a/src/transformers/utils/hub.py +++ b/src/transformers/utils/hub.py @@ -98,7 +98,7 @@ S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" _staging_mode = os.environ.get("HUGGINGFACE_CO_STAGING", "NO").upper() in ENV_VARS_TRUE_VALUES -_default_endpoint = "https://moon-staging.huggingface.co" if _staging_mode else "https://huggingface.co" +_default_endpoint = "https://hub-ci.huggingface.co" if _staging_mode else "https://huggingface.co" HUGGINGFACE_CO_RESOLVE_ENDPOINT = _default_endpoint if os.environ.get("HUGGINGFACE_CO_RESOLVE_ENDPOINT", None) is not None: diff --git a/tests/models/auto/test_processor_auto.py b/tests/models/auto/test_processor_auto.py index 26122e6164..2f99d5c379 100644 --- a/tests/models/auto/test_processor_auto.py +++ b/tests/models/auto/test_processor_auto.py @@ -21,7 +21,7 @@ import unittest from pathlib import Path from shutil import copyfile -from huggingface_hub import Repository, delete_repo, login +from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import ( CONFIG_MAPPING, @@ -36,7 +36,7 @@ from transformers import ( Wav2Vec2FeatureExtractor, Wav2Vec2Processor, ) -from transformers.testing_utils import PASS, USER, get_tests_dir, is_staging_test +from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available @@ -209,22 +209,24 @@ class ProcessorPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-processor") + delete_repo(token=cls._token, repo_id="test-processor") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-processor-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-processor-org") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-dynamic-processor") + delete_repo(token=cls._token, repo_id="test-dynamic-processor") except HTTPError: pass diff --git a/tests/test_configuration_common.py b/tests/test_configuration_common.py index 93723d11ba..4fc75b6e1d 100644 --- a/tests/test_configuration_common.py +++ b/tests/test_configuration_common.py @@ -23,11 +23,11 @@ import unittest import unittest.mock as mock from pathlib import Path -from huggingface_hub import Repository, delete_repo, login +from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import AutoConfig, BertConfig, GPT2Config, is_torch_available from transformers.configuration_utils import PretrainedConfig -from transformers.testing_utils import PASS, USER, is_staging_test +from transformers.testing_utils import TOKEN, USER, is_staging_test sys.path.append(str(Path(__file__).parent.parent / "utils")) @@ -205,22 +205,24 @@ class ConfigTester(object): class ConfigPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-config") + delete_repo(token=cls._token, repo_id="test-config") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-config-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-config-org") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-dynamic-config") + delete_repo(token=cls._token, repo_id="test-dynamic-config") except HTTPError: pass diff --git a/tests/test_feature_extraction_common.py b/tests/test_feature_extraction_common.py index 4de2cb3b8b..d6420ad3c5 100644 --- a/tests/test_feature_extraction_common.py +++ b/tests/test_feature_extraction_common.py @@ -22,10 +22,10 @@ import unittest import unittest.mock as mock from pathlib import Path -from huggingface_hub import Repository, delete_repo, login +from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import AutoFeatureExtractor, Wav2Vec2FeatureExtractor -from transformers.testing_utils import PASS, USER, check_json_file_has_correct_format, get_tests_dir, is_staging_test +from transformers.testing_utils import TOKEN, USER, check_json_file_has_correct_format, get_tests_dir, is_staging_test from transformers.utils import is_torch_available, is_vision_available @@ -139,22 +139,24 @@ class FeatureExtractorUtilTester(unittest.TestCase): class FeatureExtractorPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-feature-extractor") + delete_repo(token=cls._token, repo_id="test-feature-extractor") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-feature-extractor-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-feature-extractor-org") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-dynamic-feature-extractor") + delete_repo(token=cls._token, repo_id="test-dynamic-feature-extractor") except HTTPError: pass diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 7bdb4a0590..9a7fdea5a3 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -32,7 +32,7 @@ from typing import Dict, List, Tuple import numpy as np import transformers -from huggingface_hub import Repository, delete_repo, login +from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import ( AutoConfig, @@ -44,7 +44,7 @@ from transformers import ( ) from transformers.models.auto import get_values from transformers.testing_utils import ( - PASS, + TOKEN, USER, CaptureLogger, TestCasePlus, @@ -2782,27 +2782,29 @@ class ModelUtilsTest(TestCasePlus): class ModelPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-model") + delete_repo(token=cls._token, repo_id="test-model") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-model-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-model-org") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-dynamic-model") + delete_repo(token=cls._token, repo_id="test-dynamic-model") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-dynamic-model-config") + delete_repo(token=cls._token, repo_id="test-dynamic-model-config") except HTTPError: pass diff --git a/tests/test_modeling_flax_common.py b/tests/test_modeling_flax_common.py index b4238facc1..046dab2040 100644 --- a/tests/test_modeling_flax_common.py +++ b/tests/test_modeling_flax_common.py @@ -22,12 +22,12 @@ from typing import List, Tuple import numpy as np import transformers -from huggingface_hub import delete_repo, login +from huggingface_hub import HfFolder, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import BertConfig, is_flax_available, is_torch_available from transformers.models.auto import get_values from transformers.testing_utils import ( - PASS, + TOKEN, USER, CaptureLogger, is_pt_flax_cross_test, @@ -997,17 +997,19 @@ class FlaxModelTesterMixin: class FlaxModelPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-model-flax") + delete_repo(token=cls._token, repo_id="test-model-flax") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-model-flax-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-model-flax-org") except HTTPError: pass diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 3a8a9c80cf..b9f51a662c 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -27,14 +27,14 @@ from typing import List, Tuple from datasets import Dataset -from huggingface_hub import delete_repo, login +from huggingface_hub import HfFolder, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import is_tf_available, is_torch_available from transformers.configuration_utils import PretrainedConfig from transformers.models.auto import get_values from transformers.testing_utils import tooslow # noqa: F401 from transformers.testing_utils import ( - PASS, + TOKEN, USER, CaptureLogger, _tf_gpu_memory_limit, @@ -1873,17 +1873,19 @@ class UtilsFunctionsTest(unittest.TestCase): class TFModelPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-model-tf") + delete_repo(token=cls._token, repo_id="test-model-tf") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-model-tf-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-model-tf-org") except HTTPError: pass @@ -1911,7 +1913,7 @@ class TFModelPushToHubTester(unittest.TestCase): model = TFBertModel(config) with tempfile.TemporaryDirectory() as tmp_dir: model.push_to_hub(os.path.join(tmp_dir, "test-model-tf")) - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "test-model-card-tf", "README.md"))) + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "test-model-tf", "README.md"))) def test_push_to_hub_in_organization(self): config = BertConfig( diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 050f9a6f5d..4b27c0edb7 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -30,7 +30,7 @@ from itertools import takewhile from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union -from huggingface_hub import Repository, delete_repo, login +from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token from requests.exceptions import HTTPError from transformers import ( AlbertTokenizer, @@ -49,7 +49,7 @@ from transformers import ( is_torch_available, ) from transformers.testing_utils import ( - PASS, + TOKEN, USER, check_json_file_has_correct_format, get_tests_dir, @@ -3824,22 +3824,24 @@ class TokenizerPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): try: - delete_repo(token=cls._token, name="test-tokenizer") + delete_repo(token=cls._token, repo_id="test-tokenizer") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-tokenizer-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-tokenizer-org") except HTTPError: pass try: - delete_repo(token=cls._token, name="test-dynamic-tokenizer") + delete_repo(token=cls._token, repo_id="test-dynamic-tokenizer") except HTTPError: pass diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 9a9b3bfd88..ce58d6adeb 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -30,7 +30,7 @@ from unittest.mock import Mock, patch import numpy as np -from huggingface_hub import Repository, delete_repo, login +from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token from parameterized import parameterized from requests.exceptions import HTTPError from transformers import ( @@ -43,7 +43,7 @@ from transformers import ( ) from transformers.testing_utils import ( ENDPOINT_STAGING, - PASS, + TOKEN, USER, CaptureLogger, TestCasePlus, @@ -1966,18 +1966,20 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon): class TrainerIntegrationWithHubTester(unittest.TestCase): @classmethod def setUpClass(cls): - cls._token = login(username=USER, password=PASS) + cls._token = TOKEN + set_access_token(TOKEN) + HfFolder.save_token(TOKEN) @classmethod def tearDownClass(cls): for model in ["test-trainer", "test-trainer-epoch", "test-trainer-step"]: try: - delete_repo(token=cls._token, name=model) + delete_repo(token=cls._token, repo_id=model) except HTTPError: pass try: - delete_repo(token=cls._token, name="test-trainer-org", organization="valid_org") + delete_repo(token=cls._token, repo_id="valid_org/test-trainer-org") except HTTPError: pass