SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Prepare transformers for v0.8.0 huggingface-hub release (#17716)

Browse Source 
* Prepare CI for v0.8.0

* pin hfh (revert before merge)

* Revert "pin hfh (revert before merge)"

This reverts commit a0103140e1c77b810ffcb735192968bc03be3e1f.

* Test rc3

* Test latest rc

* Unpin to the RC

Co-authored-by: Sylvain Gugger <Sylvain.gugger@gmail.com>
This commit is contained in:
Lysandre Debut
2022-06-21 17:51:18 +02:00
committed by GitHub
parent 7bc88c0511
commit 6a5272b205
11 changed files with 75 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/transformers/commands/user.py
View File

@@ -14,10 +14,9 @@
import subprocess import subprocess
from argparse import ArgumentParser from argparse import ArgumentParser
from getpass import getpass
from typing import List, Union from typing import List, Union
from huggingface_hub.hf_api import HfFolder, create_repo, login, logout, whoami from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from . import BaseTransformersCLICommand from . import BaseTransformersCLICommand
@@ -106,33 +105,11 @@ class LoginCommand(BaseUserCommand):
def run(self): def run(self):
print( print(
ANSI.red( ANSI.red(
"WARNING! `transformers-cli login` is deprecated and will be removed in v5. Please use " "ERROR! `transformers-cli login` uses an outdated login mechanism "
"`huggingface-cli login` instead." "that is not compatible with the Hugging Face Hub backend anymore. "
"Please use `huggingface-cli login instead."
) )
) )
print( # docstyle-ignore
"""
_| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|
_| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|
_|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|
_| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|
_| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|
"""
)
username = input("Username: ")
password = getpass()
try:
token = login(username, password)
except HTTPError as e:
# probably invalid credentials, display error message.
print(e)
print(ANSI.red(e.response.text))
exit(1)
HfFolder.save_token(token)
print("Login successful")
print("Your token:", token, "\n")
print("Your token has been saved to", HfFolder.path_token)
class WhoamiCommand(BaseUserCommand): class WhoamiCommand(BaseUserCommand):
@@ -162,17 +139,11 @@ class LogoutCommand(BaseUserCommand):
def run(self): def run(self):
print( print(
ANSI.red( ANSI.red(
"WARNING! `transformers-cli logout` is deprecated and will be removed in v5. Please use " "ERROR! `transformers-cli logout` uses an outdated logout mechanism "
"`huggingface-cli logout` instead." "that is not compatible with the Hugging Face Hub backend anymore. "
"Please use `huggingface-cli logout instead."
) )
) )
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
HfFolder.delete_token()
logout(token)
print("Successfully logged out.")
class RepoCreateCommand(BaseUserCommand): class RepoCreateCommand(BaseUserCommand):

6
src/transformers/testing_utils.py
View File

@@ -84,8 +84,10 @@ DUMMY_DIFF_TOKENIZER_IDENTIFIER = "julien-c/dummy-diff-tokenizer"
# Used to test the hub # Used to test the hub
USER = "__DUMMY_TRANSFORMERS_USER__" USER = "__DUMMY_TRANSFORMERS_USER__"
PASS = "__DUMMY_TRANSFORMERS_PASS__" ENDPOINT_STAGING = "https://hub-ci.huggingface.co"
ENDPOINT_STAGING = "https://moon-staging.huggingface.co"
# Not critical, only usable on the sandboxed CI instance.
TOKEN = "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL"
def parse_flag_from_env(key, default=False): def parse_flag_from_env(key, default=False):

2
src/transformers/utils/hub.py
View File

@@ -98,7 +98,7 @@ S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert"
CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co"
_staging_mode = os.environ.get("HUGGINGFACE_CO_STAGING", "NO").upper() in ENV_VARS_TRUE_VALUES _staging_mode = os.environ.get("HUGGINGFACE_CO_STAGING", "NO").upper() in ENV_VARS_TRUE_VALUES
_default_endpoint = "https://moon-staging.huggingface.co" if _staging_mode else "https://huggingface.co" _default_endpoint = "https://hub-ci.huggingface.co" if _staging_mode else "https://huggingface.co"
HUGGINGFACE_CO_RESOLVE_ENDPOINT = _default_endpoint HUGGINGFACE_CO_RESOLVE_ENDPOINT = _default_endpoint
if os.environ.get("HUGGINGFACE_CO_RESOLVE_ENDPOINT", None) is not None: if os.environ.get("HUGGINGFACE_CO_RESOLVE_ENDPOINT", None) is not None:

14
tests/models/auto/test_processor_auto.py
View File

@@ -21,7 +21,7 @@ import unittest
from pathlib import Path from pathlib import Path
from shutil import copyfile from shutil import copyfile
from huggingface_hub import Repository, delete_repo, login from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import ( from transformers import (
CONFIG_MAPPING, CONFIG_MAPPING,
@@ -36,7 +36,7 @@ from transformers import (
Wav2Vec2FeatureExtractor, Wav2Vec2FeatureExtractor,
Wav2Vec2Processor, Wav2Vec2Processor,
) )
from transformers.testing_utils import PASS, USER, get_tests_dir, is_staging_test from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available
@@ -209,22 +209,24 @@ class ProcessorPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-processor") delete_repo(token=cls._token, repo_id="test-processor")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-processor-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-processor-org")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-dynamic-processor") delete_repo(token=cls._token, repo_id="test-dynamic-processor")
except HTTPError: except HTTPError:
pass pass

14
tests/test_configuration_common.py
View File

@@ -23,11 +23,11 @@ import unittest
import unittest.mock as mock import unittest.mock as mock
from pathlib import Path from pathlib import Path
from huggingface_hub import Repository, delete_repo, login from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import AutoConfig, BertConfig, GPT2Config, is_torch_available from transformers import AutoConfig, BertConfig, GPT2Config, is_torch_available
from transformers.configuration_utils import PretrainedConfig from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import PASS, USER, is_staging_test from transformers.testing_utils import TOKEN, USER, is_staging_test
sys.path.append(str(Path(__file__).parent.parent / "utils")) sys.path.append(str(Path(__file__).parent.parent / "utils"))
@@ -205,22 +205,24 @@ class ConfigTester(object):
class ConfigPushToHubTester(unittest.TestCase): class ConfigPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-config") delete_repo(token=cls._token, repo_id="test-config")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-config-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-config-org")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-dynamic-config") delete_repo(token=cls._token, repo_id="test-dynamic-config")
except HTTPError: except HTTPError:
pass pass

14
tests/test_feature_extraction_common.py
View File

@@ -22,10 +22,10 @@ import unittest
import unittest.mock as mock import unittest.mock as mock
from pathlib import Path from pathlib import Path
from huggingface_hub import Repository, delete_repo, login from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import AutoFeatureExtractor, Wav2Vec2FeatureExtractor from transformers import AutoFeatureExtractor, Wav2Vec2FeatureExtractor
from transformers.testing_utils import PASS, USER, check_json_file_has_correct_format, get_tests_dir, is_staging_test from transformers.testing_utils import TOKEN, USER, check_json_file_has_correct_format, get_tests_dir, is_staging_test
from transformers.utils import is_torch_available, is_vision_available from transformers.utils import is_torch_available, is_vision_available
@@ -139,22 +139,24 @@ class FeatureExtractorUtilTester(unittest.TestCase):
class FeatureExtractorPushToHubTester(unittest.TestCase): class FeatureExtractorPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-feature-extractor") delete_repo(token=cls._token, repo_id="test-feature-extractor")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-feature-extractor-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-feature-extractor-org")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-dynamic-feature-extractor") delete_repo(token=cls._token, repo_id="test-dynamic-feature-extractor")
except HTTPError: except HTTPError:
pass pass

16
tests/test_modeling_common.py
View File

@@ -32,7 +32,7 @@ from typing import Dict, List, Tuple
import numpy as np import numpy as np
import transformers import transformers
from huggingface_hub import Repository, delete_repo, login from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import ( from transformers import (
AutoConfig, AutoConfig,
@@ -44,7 +44,7 @@ from transformers import (
) )
from transformers.models.auto import get_values from transformers.models.auto import get_values
from transformers.testing_utils import ( from transformers.testing_utils import (
PASS, TOKEN,
USER, USER,
CaptureLogger, CaptureLogger,
TestCasePlus, TestCasePlus,
@@ -2782,27 +2782,29 @@ class ModelUtilsTest(TestCasePlus):
class ModelPushToHubTester(unittest.TestCase): class ModelPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-model") delete_repo(token=cls._token, repo_id="test-model")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-model-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-model-org")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-dynamic-model") delete_repo(token=cls._token, repo_id="test-dynamic-model")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-dynamic-model-config") delete_repo(token=cls._token, repo_id="test-dynamic-model-config")
except HTTPError: except HTTPError:
pass pass

12
tests/test_modeling_flax_common.py
View File

@@ -22,12 +22,12 @@ from typing import List, Tuple
import numpy as np import numpy as np
import transformers import transformers
from huggingface_hub import delete_repo, login from huggingface_hub import HfFolder, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import BertConfig, is_flax_available, is_torch_available from transformers import BertConfig, is_flax_available, is_torch_available
from transformers.models.auto import get_values from transformers.models.auto import get_values
from transformers.testing_utils import ( from transformers.testing_utils import (
PASS, TOKEN,
USER, USER,
CaptureLogger, CaptureLogger,
is_pt_flax_cross_test, is_pt_flax_cross_test,
@@ -997,17 +997,19 @@ class FlaxModelTesterMixin:
class FlaxModelPushToHubTester(unittest.TestCase): class FlaxModelPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-model-flax") delete_repo(token=cls._token, repo_id="test-model-flax")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-model-flax-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-model-flax-org")
except HTTPError: except HTTPError:
pass pass

14
tests/test_modeling_tf_common.py
View File

@@ -27,14 +27,14 @@ from typing import List, Tuple
from datasets import Dataset from datasets import Dataset
from huggingface_hub import delete_repo, login from huggingface_hub import HfFolder, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import is_tf_available, is_torch_available from transformers import is_tf_available, is_torch_available
from transformers.configuration_utils import PretrainedConfig from transformers.configuration_utils import PretrainedConfig
from transformers.models.auto import get_values from transformers.models.auto import get_values
from transformers.testing_utils import tooslow # noqa: F401 from transformers.testing_utils import tooslow # noqa: F401
from transformers.testing_utils import ( from transformers.testing_utils import (
PASS, TOKEN,
USER, USER,
CaptureLogger, CaptureLogger,
_tf_gpu_memory_limit, _tf_gpu_memory_limit,
@@ -1873,17 +1873,19 @@ class UtilsFunctionsTest(unittest.TestCase):
class TFModelPushToHubTester(unittest.TestCase): class TFModelPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-model-tf") delete_repo(token=cls._token, repo_id="test-model-tf")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-model-tf-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-model-tf-org")
except HTTPError: except HTTPError:
pass pass
@@ -1911,7 +1913,7 @@ class TFModelPushToHubTester(unittest.TestCase):
model = TFBertModel(config) model = TFBertModel(config)
with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.TemporaryDirectory() as tmp_dir:
model.push_to_hub(os.path.join(tmp_dir, "test-model-tf")) model.push_to_hub(os.path.join(tmp_dir, "test-model-tf"))
self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "test-model-card-tf", "README.md"))) self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "test-model-tf", "README.md")))
def test_push_to_hub_in_organization(self): def test_push_to_hub_in_organization(self):
config = BertConfig( config = BertConfig(

14
tests/test_tokenization_common.py
View File

@@ -30,7 +30,7 @@ from itertools import takewhile
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union
from huggingface_hub import Repository, delete_repo, login from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import ( from transformers import (
AlbertTokenizer, AlbertTokenizer,
@@ -49,7 +49,7 @@ from transformers import (
is_torch_available, is_torch_available,
) )
from transformers.testing_utils import ( from transformers.testing_utils import (
PASS, TOKEN,
USER, USER,
check_json_file_has_correct_format, check_json_file_has_correct_format,
get_tests_dir, get_tests_dir,
@@ -3824,22 +3824,24 @@ class TokenizerPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
delete_repo(token=cls._token, name="test-tokenizer") delete_repo(token=cls._token, repo_id="test-tokenizer")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-tokenizer-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-tokenizer-org")
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-dynamic-tokenizer") delete_repo(token=cls._token, repo_id="test-dynamic-tokenizer")
except HTTPError: except HTTPError:
pass pass

12
tests/trainer/test_trainer.py
View File

@@ -30,7 +30,7 @@ from unittest.mock import Mock, patch
import numpy as np import numpy as np
from huggingface_hub import Repository, delete_repo, login from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
from parameterized import parameterized from parameterized import parameterized
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from transformers import ( from transformers import (
@@ -43,7 +43,7 @@ from transformers import (
) )
from transformers.testing_utils import ( from transformers.testing_utils import (
ENDPOINT_STAGING, ENDPOINT_STAGING,
PASS, TOKEN,
USER, USER,
CaptureLogger, CaptureLogger,
TestCasePlus, TestCasePlus,
@@ -1966,18 +1966,20 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
class TrainerIntegrationWithHubTester(unittest.TestCase): class TrainerIntegrationWithHubTester(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._token = login(username=USER, password=PASS) cls._token = TOKEN
set_access_token(TOKEN)
HfFolder.save_token(TOKEN)
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
for model in ["test-trainer", "test-trainer-epoch", "test-trainer-step"]: for model in ["test-trainer", "test-trainer-epoch", "test-trainer-step"]:
try: try:
delete_repo(token=cls._token, name=model) delete_repo(token=cls._token, repo_id=model)
except HTTPError: except HTTPError:
pass pass
try: try:
delete_repo(token=cls._token, name="test-trainer-org", organization="valid_org") delete_repo(token=cls._token, repo_id="valid_org/test-trainer-org")
except HTTPError: except HTTPError:
pass pass