Better logic for getting tokenizer config in AutoTokenizer (#14906)

* Better logic for getting tokenizer config in AutoTokenizer

* Remove needless import

* Remove debug statement

* Address review comments
This commit is contained in:
Sylvain Gugger
2021-12-23 14:18:07 -05:00
committed by GitHub
parent f566c6e3b7
commit 676643c6d6
3 changed files with 24 additions and 3 deletions

View File

@@ -149,7 +149,9 @@ class AutoTokenizerTest(unittest.TestCase):
@require_tokenizers
def test_tokenizer_identifier_non_existent(self):
for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
with self.assertRaises(EnvironmentError):
with self.assertRaisesRegex(
ValueError, ".*is not a local path or a model identifier on the model Hub. Did you make a typo?"
):
_ = tokenizer_class.from_pretrained("julien-c/herlolip-not-exists")
def test_parents_and_children_in_mappings(self):