SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix add_special_tokens on fast tokenizers (#4531)

Browse Source
This commit is contained in:
Anthony MOI
2020-05-28 10:54:45 -04:00
committed by GitHub
parent e444648a30
commit 5e737018e1
2 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/test_tokenization_fast.py
View File

@@ -221,6 +221,7 @@ class CommonFastTokenizerTest(unittest.TestCase):
self.assertEqual(len(tokenizer_r), vocab_size + 3)
self.assertEqual(tokenizer_r.add_special_tokens({}), 0)
self.assertEqual(tokenizer_r.add_special_tokens({"bos_token": "[BOS]", "eos_token": "[EOS]"}), 2)
self.assertRaises(
AssertionError, tokenizer_r.add_special_tokens, {"additional_special_tokens": "<testtoken1>"}
)
@@ -228,7 +229,7 @@ class CommonFastTokenizerTest(unittest.TestCase):
self.assertEqual(
tokenizer_r.add_special_tokens({"additional_special_tokens": ["<testtoken3>", "<testtoken4>"]}), 2
)
self.assertEqual(len(tokenizer_r), vocab_size + 6)
self.assertEqual(len(tokenizer_r), vocab_size + 8)
def assert_offsets_mapping(self, tokenizer_r):
text = "Wonderful no inspiration example with subtoken"