SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Update Transformers to huggingface_hub >= 0.1.0 (#14251)

Browse Source 
* Update Transformers to huggingface_hub >= 0.1.0

* Forgot to save...

* Style

* Fix test
This commit is contained in:
Sylvain Gugger
2021-11-02 18:58:42 -04:00
committed by GitHub
parent 519a677e87
commit 558f8543ba
15 changed files with 70 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tests/test_configuration_common.py
View File

@@ -19,11 +19,11 @@ import os
import tempfile
import unittest
from huggingface_hub import HfApi
from huggingface_hub import delete_repo, login
from requests.exceptions import HTTPError
from transformers import BertConfig, GPT2Config, is_torch_available
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import ENDPOINT_STAGING, PASS, USER, is_staging_test
from transformers.testing_utils import PASS, USER, is_staging_test
config_common_kwargs = {
@@ -194,18 +194,17 @@ class ConfigTester(object):
class ConfigPushToHubTester(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._api = HfApi(endpoint=ENDPOINT_STAGING)
cls._token = cls._api.login(username=USER, password=PASS)
cls._token = login(username=USER, password=PASS)
@classmethod
def tearDownClass(cls):
try:
cls._api.delete_repo(token=cls._token, name="test-config")
delete_repo(token=cls._token, name="test-config")
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-config-org", organization="valid_org")
delete_repo(token=cls._token, name="test-config-org", organization="valid_org")
except HTTPError:
pass

12
tests/test_modeling_common.py
View File

@@ -28,13 +28,12 @@ from typing import Dict, List, Tuple
import numpy as np
import transformers
from huggingface_hub import HfApi, Repository
from huggingface_hub import Repository, delete_repo, login
from requests.exceptions import HTTPError
from transformers import AutoModel, AutoModelForSequenceClassification, is_torch_available, logging
from transformers.file_utils import WEIGHTS_NAME, is_flax_available, is_torch_fx_available
from transformers.models.auto import get_values
from transformers.testing_utils import (
ENDPOINT_STAGING,
PASS,
USER,
CaptureLogger,
@@ -2122,23 +2121,22 @@ class FakeModel(PreTrainedModel):
class ModelPushToHubTester(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._api = HfApi(endpoint=ENDPOINT_STAGING)
cls._token = cls._api.login(username=USER, password=PASS)
cls._token = login(username=USER, password=PASS)
@classmethod
def tearDownClass(cls):
try:
cls._api.delete_repo(token=cls._token, name="test-model")
delete_repo(token=cls._token, name="test-model")
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-model-org", organization="valid_org")
delete_repo(token=cls._token, name="test-model-org", organization="valid_org")
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-dynamic-model")
delete_repo(token=cls._token, name="test-dynamic-model")
except HTTPError:
pass

19
tests/test_modeling_flax_common.py
View File

@@ -22,19 +22,11 @@ from typing import List, Tuple
import numpy as np
import transformers
from huggingface_hub import HfApi
from huggingface_hub import delete_repo, login
from requests.exceptions import HTTPError
from transformers import BertConfig, is_flax_available, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import (
ENDPOINT_STAGING,
PASS,
USER,
CaptureLogger,
is_pt_flax_cross_test,
is_staging_test,
require_flax,
)
from transformers.testing_utils import PASS, USER, CaptureLogger, is_pt_flax_cross_test, is_staging_test, require_flax
from transformers.utils import logging
@@ -627,18 +619,17 @@ class FlaxModelTesterMixin:
class FlaxModelPushToHubTester(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._api = HfApi(endpoint=ENDPOINT_STAGING)
cls._token = cls._api.login(username=USER, password=PASS)
cls._token = login(username=USER, password=PASS)
@classmethod
def tearDownClass(cls):
try:
cls._api.delete_repo(token=cls._token, name="test-model-flax")
delete_repo(token=cls._token, name="test-model-flax")
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-model-flax-org", organization="valid_org")
delete_repo(token=cls._token, name="test-model-flax-org", organization="valid_org")
except HTTPError:
pass

4
tests/test_modeling_marian.py
View File

@@ -17,7 +17,7 @@
import tempfile
import unittest
from huggingface_hub.hf_api import HfApi
from huggingface_hub.hf_api import list_models
from transformers import MarianConfig, is_torch_available
from transformers.file_utils import cached_property
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
@@ -296,7 +296,7 @@ class ModelManagementTests(unittest.TestCase):
@slow
@require_torch
def test_model_names(self):
model_list = HfApi().list_models()
model_list = list_models()
model_ids = [x.modelId for x in model_list if x.modelId.startswith(ORG_NAME)]
bad_model_ids = [mid for mid in model_ids if "+" in model_ids]
self.assertListEqual([], bad_model_ids)

10
tests/test_modeling_tf_common.py
View File

@@ -24,12 +24,11 @@ import unittest
from importlib import import_module
from typing import List, Tuple
from huggingface_hub import HfApi
from huggingface_hub import delete_repo, login
from requests.exceptions import HTTPError
from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import (
ENDPOINT_STAGING,
PASS,
USER,
CaptureLogger,
@@ -1530,18 +1529,17 @@ class UtilsFunctionsTest(unittest.TestCase):
class TFModelPushToHubTester(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._api = HfApi(endpoint=ENDPOINT_STAGING)
cls._token = cls._api.login(username=USER, password=PASS)
cls._token = login(username=USER, password=PASS)
@classmethod
def tearDownClass(cls):
try:
cls._api.delete_repo(token=cls._token, name="test-model-tf")
delete_repo(token=cls._token, name="test-model-tf")
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-model-tf-org", organization="valid_org")
delete_repo(token=cls._token, name="test-model-tf-org", organization="valid_org")
except HTTPError:
pass

10
tests/test_tokenization_common.py
View File

@@ -27,7 +27,7 @@ from collections import OrderedDict
from itertools import takewhile
from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union
from huggingface_hub import HfApi
from huggingface_hub import delete_repo, login
from requests.exceptions import HTTPError
from transformers import (
AlbertTokenizer,
@@ -44,7 +44,6 @@ from transformers import (
is_torch_available,
)
from transformers.testing_utils import (
ENDPOINT_STAGING,
PASS,
USER,
get_tests_dir,
@@ -3520,18 +3519,17 @@ class TokenizerPushToHubTester(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._api = HfApi(endpoint=ENDPOINT_STAGING)
cls._token = cls._api.login(username=USER, password=PASS)
cls._token = login(username=USER, password=PASS)
@classmethod
def tearDownClass(cls):
try:
cls._api.delete_repo(token=cls._token, name="test-tokenizer")
delete_repo(token=cls._token, name="test-tokenizer")
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-tokenizer-org", organization="valid_org")
delete_repo(token=cls._token, name="test-tokenizer-org", organization="valid_org")
except HTTPError:
pass

16
tests/test_trainer.py
View File

@@ -26,7 +26,7 @@ from pathlib import Path
import numpy as np
from huggingface_hub import HfApi, Repository
from huggingface_hub import Repository, delete_repo, login
from requests.exceptions import HTTPError
from transformers import (
AutoTokenizer,
@@ -1307,19 +1307,18 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
class TrainerIntegrationWithHubTester(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._api = HfApi(endpoint=ENDPOINT_STAGING)
cls._token = cls._api.login(username=USER, password=PASS)
cls._token = login(username=USER, password=PASS)
@classmethod
def tearDownClass(cls):
for model in ["test-trainer", "test-trainer-epoch", "test-trainer-step"]:
try:
cls._api.delete_repo(token=cls._token, name=model)
delete_repo(token=cls._token, name=model)
except HTTPError:
pass
try:
cls._api.delete_repo(token=cls._token, name="test-trainer-org", organization="valid_org")
delete_repo(token=cls._token, name="test-trainer-org", organization="valid_org")
except HTTPError:
pass
@@ -1396,6 +1395,10 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
print(commits, len(commits))
def test_push_to_hub_with_saves_each_n_steps(self):
num_gpus = max(1, get_gpu_count())
if num_gpus > 2:
return
with tempfile.TemporaryDirectory() as tmp_dir:
trainer = get_regression_trainer(
output_dir=os.path.join(tmp_dir, "test-trainer-step"),
@@ -1409,7 +1412,8 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp_dir:
_ = Repository(tmp_dir, clone_from=f"{USER}/test-trainer-step", use_auth_token=self._token)
commits = self.get_commit_history(tmp_dir)
expected_commits = [f"Training in progress, step {i}" for i in range(20, 0, -5)]
total_steps = 20 // num_gpus
expected_commits = [f"Training in progress, step {i}" for i in range(total_steps, 0, -5)]
expected_commits.append("initial commit")
self.assertListEqual(commits, expected_commits)
print(commits, len(commits))