SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

More AutoConfig tests

Browse Source
This commit is contained in:
Julien Chaumond
2020-01-11 03:43:57 +00:00
parent 6bb3edc300
commit cf8a70bf68
5 changed files with 35 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tests/test_tokenization_auto.py
View File

@@ -23,9 +23,10 @@ from transformers import (
AutoTokenizer,
BertTokenizer,
GPT2Tokenizer,
RobertaTokenizer,
)
from .utils import SMALL_MODEL_IDENTIFIER, slow # noqa: F401
from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, slow # noqa: F401
class AutoTokenizerTest(unittest.TestCase):
@@ -49,3 +50,9 @@ class AutoTokenizerTest(unittest.TestCase):
tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER)
self.assertIsInstance(tokenizer, BertTokenizer)
self.assertEqual(len(tokenizer), 12)
def test_tokenizer_from_model_type(self):
logging.basicConfig(level=logging.INFO)
tokenizer = AutoTokenizer.from_pretrained(DUMMY_UNKWOWN_IDENTIFIER)
self.assertIsInstance(tokenizer, RobertaTokenizer)
self.assertEqual(len(tokenizer), 20)