SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix BasicTokenizer to respect never_split parameters (#2557)

Browse Source 
* add failing test

* fix call to _run_split_on_punc

* format with black
This commit is contained in:
Mark Neumann
2020-01-17 11:57:56 -08:00
committed by Julien Chaumond
parent 6d5049a24d
commit 65a89a8976
2 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/test_tokenization_bert.py
View File

@@ -119,6 +119,13 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), ["HeLLo", "!", "how", "Are", "yoU", "?"]
)
def test_basic_tokenizer_respects_never_split_tokens(self):
tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"])
self.assertListEqual(
tokenizer.tokenize(" \tHeLLo!how \n Are yoU? [UNK]"), ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]
)
def test_wordpiece_tokenizer(self):
vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]