From 65a89a89768f5922e51cdc7d49990d731e3f2c03 Mon Sep 17 00:00:00 2001 From: Mark Neumann Date: Fri, 17 Jan 2020 11:57:56 -0800 Subject: [PATCH] Fix BasicTokenizer to respect `never_split` parameters (#2557) * add failing test * fix call to _run_split_on_punc * format with black --- src/transformers/tokenization_bert.py | 2 +- tests/test_tokenization_bert.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_bert.py b/src/transformers/tokenization_bert.py index ac14370a18..cefb917798 100644 --- a/src/transformers/tokenization_bert.py +++ b/src/transformers/tokenization_bert.py @@ -341,7 +341,7 @@ class BasicTokenizer(object): if self.do_lower_case and token not in never_split: token = token.lower() token = self._run_strip_accents(token) - split_tokens.extend(self._run_split_on_punc(token)) + split_tokens.extend(self._run_split_on_punc(token, never_split)) output_tokens = whitespace_tokenize(" ".join(split_tokens)) return output_tokens diff --git a/tests/test_tokenization_bert.py b/tests/test_tokenization_bert.py index 793bb8fa54..49bb073351 100644 --- a/tests/test_tokenization_bert.py +++ b/tests/test_tokenization_bert.py @@ -119,6 +119,13 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase): tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), ["HeLLo", "!", "how", "Are", "yoU", "?"] ) + def test_basic_tokenizer_respects_never_split_tokens(self): + tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"]) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? [UNK]"), ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] + ) + def test_wordpiece_tokenizer(self): vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]