SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Test correct tokenizers after default switch (#3003)

Browse Source
This commit is contained in:
Lysandre Debut
2020-02-24 18:45:53 -05:00
committed by GitHub
parent 3716c3d8af
commit b90745c590
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_tokenization_auto.py
View File

@@ -101,5 +101,5 @@ class AutoTokenizerTest(unittest.TestCase):
self.assertFalse(issubclass(child_model_fast, parent_model_fast)) self.assertFalse(issubclass(child_model_fast, parent_model_fast))
def test_from_pretrained_use_fast_toggle(self): def test_from_pretrained_use_fast_toggle(self):
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizerFast) self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizer)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False), BertTokenizer) self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=True), BertTokenizerFast)