SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add tests for legacy load by url and fix bugs (#19078)

Browse Source
This commit is contained in:
Sylvain Gugger
2022-09-16 17:20:02 -04:00
committed by GitHub
parent ae219532e3
commit ca485e562b
9 changed files with 62 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/test_tokenization_common.py
View File

@@ -3891,15 +3891,20 @@ class TokenizerUtilTester(unittest.TestCase):
mock_head.assert_called()
def test_legacy_load_from_one_file(self):
# This test is for deprecated behavior and can be removed in v5
try:
tmp_file = tempfile.mktemp()
with open(tmp_file, "wb") as f:
http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model", f)
AlbertTokenizer.from_pretrained(tmp_file)
_ = AlbertTokenizer.from_pretrained(tmp_file)
finally:
os.remove(tmp_file)
def test_legacy_load_from_url(self):
# This test is for deprecated behavior and can be removed in v5
_ = AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model")
@is_staging_test
class TokenizerPushToHubTester(unittest.TestCase):