Fix flaky Hub CI (test_trainer.py) (#35062)
* fix * Update src/transformers/testing_utils.py Co-authored-by: Lucain <lucainp@gmail.com> * fix * fix * fix * fix * fix * fix * fix * fix * check * check * check * check * check * check * Update src/transformers/testing_utils.py Co-authored-by: Lucain <lucainp@gmail.com> * Update src/transformers/testing_utils.py Co-authored-by: Lucain <lucainp@gmail.com> * check * check * check * Final space * Final adjustment --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com> Co-authored-by: Lucain <lucainp@gmail.com>
This commit is contained in:
Yih-Dar
@@ -28,7 +28,7 @@ import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from huggingface_hub import HfApi, HfFolder, delete_repo
|
||||
from huggingface_hub import HfApi, HfFolder
|
||||
from pytest import mark
|
||||
from requests.exceptions import HTTPError
|
||||
|
||||
@@ -44,9 +44,9 @@ from transformers import (
|
||||
)
|
||||
from transformers.testing_utils import (
|
||||
TOKEN,
|
||||
USER,
|
||||
CaptureLogger,
|
||||
LoggingLevel,
|
||||
TemporaryHubRepo,
|
||||
TestCasePlus,
|
||||
is_staging_test,
|
||||
require_accelerate,
|
||||
@@ -2000,168 +2000,127 @@ class ModelPushToHubTester(unittest.TestCase):
|
||||
cls._token = TOKEN
|
||||
HfFolder.save_token(TOKEN)
|
||||
|
||||
@staticmethod
|
||||
def _try_delete_repo(repo_id, token):
|
||||
try:
|
||||
# Reset repo
|
||||
delete_repo(repo_id=repo_id, token=token)
|
||||
except: # noqa E722
|
||||
pass
|
||||
|
||||
@unittest.skip(reason="This test is flaky")
|
||||
def test_push_to_hub(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"{USER}/test-model-{Path(tmp_dir).name}"
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
model.push_to_hub(tmp_repo, token=self._token)
|
||||
with TemporaryHubRepo(token=self._token) as tmp_repo:
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
model.push_to_hub(tmp_repo.repo_id, token=self._token)
|
||||
|
||||
new_model = BertModel.from_pretrained(tmp_repo)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
new_model = BertModel.from_pretrained(tmp_repo.repo_id)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
|
||||
@unittest.skip(reason="This test is flaky")
|
||||
def test_push_to_hub_via_save_pretrained(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"{USER}/test-model-{Path(tmp_dir).name}"
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
# Push to hub via save_pretrained
|
||||
model.save_pretrained(tmp_dir, repo_id=tmp_repo, push_to_hub=True, token=self._token)
|
||||
with TemporaryHubRepo(token=self._token) as tmp_repo:
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
# Push to hub via save_pretrained
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir, repo_id=tmp_repo.repo_id, push_to_hub=True, token=self._token)
|
||||
|
||||
new_model = BertModel.from_pretrained(tmp_repo)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
new_model = BertModel.from_pretrained(tmp_repo.repo_id)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
|
||||
def test_push_to_hub_with_description(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"{USER}/test-model-{Path(tmp_dir).name}"
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
COMMIT_DESCRIPTION = """
|
||||
with TemporaryHubRepo(token=self._token) as tmp_repo:
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
COMMIT_DESCRIPTION = """
|
||||
The commit description supports markdown synthax see:
|
||||
```python
|
||||
>>> form transformers import AutoConfig
|
||||
>>> config = AutoConfig.from_pretrained("google-bert/bert-base-uncased")
|
||||
```
|
||||
"""
|
||||
commit_details = model.push_to_hub(
|
||||
tmp_repo, use_auth_token=self._token, create_pr=True, commit_description=COMMIT_DESCRIPTION
|
||||
)
|
||||
self.assertEqual(commit_details.commit_description, COMMIT_DESCRIPTION)
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
commit_details = model.push_to_hub(
|
||||
tmp_repo.repo_id, use_auth_token=self._token, create_pr=True, commit_description=COMMIT_DESCRIPTION
|
||||
)
|
||||
self.assertEqual(commit_details.commit_description, COMMIT_DESCRIPTION)
|
||||
|
||||
@unittest.skip(reason="This test is flaky")
|
||||
def test_push_to_hub_in_organization(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"valid_org/test-model-org-{Path(tmp_dir).name}"
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
model.push_to_hub(tmp_repo, token=self._token)
|
||||
with TemporaryHubRepo(namespace="valid_org", token=self._token) as tmp_repo:
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
model.push_to_hub(tmp_repo.repo_id, token=self._token)
|
||||
|
||||
new_model = BertModel.from_pretrained(tmp_repo)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
new_model = BertModel.from_pretrained(tmp_repo.repo_id)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
|
||||
@unittest.skip(reason="This test is flaky")
|
||||
def test_push_to_hub_in_organization_via_save_pretrained(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"valid_org/test-model-org-{Path(tmp_dir).name}"
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
# Push to hub via save_pretrained
|
||||
model.save_pretrained(tmp_dir, push_to_hub=True, token=self._token, repo_id=tmp_repo)
|
||||
with TemporaryHubRepo(namespace="valid_org", token=self._token) as tmp_repo:
|
||||
config = BertConfig(
|
||||
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
|
||||
)
|
||||
model = BertModel(config)
|
||||
# Push to hub via save_pretrained
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir, push_to_hub=True, token=self._token, repo_id=tmp_repo.repo_id)
|
||||
|
||||
new_model = BertModel.from_pretrained(tmp_repo)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
new_model = BertModel.from_pretrained(tmp_repo.repo_id)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
|
||||
def test_push_to_hub_dynamic_model(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"{USER}/test-dynamic-model-{Path(tmp_dir).name}"
|
||||
CustomConfig.register_for_auto_class()
|
||||
CustomModel.register_for_auto_class()
|
||||
with TemporaryHubRepo(token=self._token) as tmp_repo:
|
||||
CustomConfig.register_for_auto_class()
|
||||
CustomModel.register_for_auto_class()
|
||||
|
||||
config = CustomConfig(hidden_size=32)
|
||||
model = CustomModel(config)
|
||||
config = CustomConfig(hidden_size=32)
|
||||
model = CustomModel(config)
|
||||
|
||||
model.push_to_hub(tmp_repo, token=self._token)
|
||||
# checks
|
||||
self.assertDictEqual(
|
||||
config.auto_map,
|
||||
{"AutoConfig": "custom_configuration.CustomConfig", "AutoModel": "custom_modeling.CustomModel"},
|
||||
)
|
||||
model.push_to_hub(tmp_repo.repo_id, token=self._token)
|
||||
# checks
|
||||
self.assertDictEqual(
|
||||
config.auto_map,
|
||||
{"AutoConfig": "custom_configuration.CustomConfig", "AutoModel": "custom_modeling.CustomModel"},
|
||||
)
|
||||
|
||||
new_model = AutoModel.from_pretrained(tmp_repo, trust_remote_code=True)
|
||||
# Can't make an isinstance check because the new_model is from the CustomModel class of a dynamic module
|
||||
self.assertEqual(new_model.__class__.__name__, "CustomModel")
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
new_model = AutoModel.from_pretrained(tmp_repo.repo_id, trust_remote_code=True)
|
||||
# Can't make an isinstance check because the new_model is from the CustomModel class of a dynamic module
|
||||
self.assertEqual(new_model.__class__.__name__, "CustomModel")
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
|
||||
config = AutoConfig.from_pretrained(tmp_repo, trust_remote_code=True)
|
||||
new_model = AutoModel.from_config(config, trust_remote_code=True)
|
||||
self.assertEqual(new_model.__class__.__name__, "CustomModel")
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
config = AutoConfig.from_pretrained(tmp_repo.repo_id, trust_remote_code=True)
|
||||
new_model = AutoModel.from_config(config, trust_remote_code=True)
|
||||
self.assertEqual(new_model.__class__.__name__, "CustomModel")
|
||||
|
||||
def test_push_to_hub_with_tags(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
try:
|
||||
tmp_repo = f"{USER}/test-dynamic-model-with-tags-{Path(tmp_dir).name}"
|
||||
from huggingface_hub import ModelCard
|
||||
with TemporaryHubRepo(token=self._token) as tmp_repo:
|
||||
from huggingface_hub import ModelCard
|
||||
|
||||
new_tags = ["tag-1", "tag-2"]
|
||||
new_tags = ["tag-1", "tag-2"]
|
||||
|
||||
CustomConfig.register_for_auto_class()
|
||||
CustomModel.register_for_auto_class()
|
||||
CustomConfig.register_for_auto_class()
|
||||
CustomModel.register_for_auto_class()
|
||||
|
||||
config = CustomConfig(hidden_size=32)
|
||||
model = CustomModel(config)
|
||||
config = CustomConfig(hidden_size=32)
|
||||
model = CustomModel(config)
|
||||
|
||||
self.assertTrue(model.model_tags is None)
|
||||
self.assertTrue(model.model_tags is None)
|
||||
|
||||
model.add_model_tags(new_tags)
|
||||
model.add_model_tags(new_tags)
|
||||
|
||||
self.assertTrue(model.model_tags == new_tags)
|
||||
self.assertTrue(model.model_tags == new_tags)
|
||||
|
||||
model.push_to_hub(tmp_repo, token=self._token)
|
||||
model.push_to_hub(tmp_repo.repo_id, token=self._token)
|
||||
|
||||
loaded_model_card = ModelCard.load(tmp_repo)
|
||||
self.assertEqual(loaded_model_card.data.tags, new_tags)
|
||||
finally:
|
||||
# Always (try to) delete the repo.
|
||||
self._try_delete_repo(repo_id=tmp_repo, token=self._token)
|
||||
loaded_model_card = ModelCard.load(tmp_repo.repo_id)
|
||||
self.assertEqual(loaded_model_card.data.tags, new_tags)
|
||||
|
||||
|
||||
@require_torch
|
||||
|
||||
Reference in New Issue
Block a user