Add sudachi and jumanpp tokenizers for bert_japanese (#19043)

* add sudachipy and jumanpp tokenizers for bert_japanese

* use ImportError instead of ModuleNotFoundError in SudachiTokenizer and JumanppTokenizer

* put test cases of test_tokenization_bert_japanese in one line

* add require_sudachi and require_jumanpp decorator for testing

* add sudachi and pyknp(jumanpp) to dependencies

* remove sudachi_dict_small and sudachi_dict_full from dependencies

* empty commit for ci

tests/models/bert_japanese/test_tokenization_bert_japanese.py

@@ -24,10 +24,12 @@ from transformers.models.bert_japanese.tokenization_bert_japanese import (
     BertJapaneseTokenizer,
     BertTokenizer,
     CharacterTokenizer,
+    JumanppTokenizer,
     MecabTokenizer,
+    SudachiTokenizer,
     WordpieceTokenizer,
 )
-from transformers.testing_utils import custom_tokenizers
+from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi
 
 from ...test_tokenization_common import TokenizerTesterMixin
 
@@ -172,6 +174,150 @@ class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
             ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"],
         )
 
+    @require_sudachi
+    def test_pickle_sudachi_tokenizer(self):
+        tokenizer = self.tokenizer_class(self.vocab_file, word_tokenizer_type="sudachi")
+        self.assertIsNotNone(tokenizer)
+
+        text = "こんにちは、世界。\nこんばんは、世界。"
+        tokens = tokenizer.tokenize(text)
+        self.assertListEqual(tokens, ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
+        self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [3, 12, 10, 14, 4, 9, 12, 10, 14])
+
+        filename = os.path.join(self.tmpdirname, "tokenizer.bin")
+        with open(filename, "wb") as handle:
+            pickle.dump(tokenizer, handle)
+
+        with open(filename, "rb") as handle:
+            tokenizer_new = pickle.load(handle)
+
+        tokens_loaded = tokenizer_new.tokenize(text)
+
+        self.assertListEqual(tokens, tokens_loaded)
+
+    @require_sudachi
+    def test_sudachi_tokenizer_core(self):
+        tokenizer = SudachiTokenizer(sudachi_dict_type="core")
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            # fmt: off
+            [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "],
+            # fmt: on
+        )
+
+    @require_sudachi
+    def test_sudachi_tokenizer_split_mode_A(self):
+        tokenizer = SudachiTokenizer(sudachi_dict_type="core", sudachi_split_mode="A")
+
+        self.assertListEqual(tokenizer.tokenize("外国人参政権"), ["外国", "人", "参政", "権"])
+
+    @require_sudachi
+    def test_sudachi_tokenizer_split_mode_B(self):
+        tokenizer = SudachiTokenizer(sudachi_dict_type="core", sudachi_split_mode="B")
+
+        self.assertListEqual(tokenizer.tokenize("外国人参政権"), ["外国人", "参政権"])
+
+    @require_sudachi
+    def test_sudachi_tokenizer_split_mode_C(self):
+        tokenizer = SudachiTokenizer(sudachi_dict_type="core", sudachi_split_mode="C")
+
+        self.assertListEqual(tokenizer.tokenize("外国人参政権"), ["外国人参政権"])
+
+    @require_sudachi
+    def test_sudachi_tokenizer_lower(self):
+        tokenizer = SudachiTokenizer(do_lower_case=True, sudachi_dict_type="core")
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            # fmt: off
+            [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "],
+            # fmt: on
+        )
+
+    @require_sudachi
+    def test_sudachi_tokenizer_no_normalize(self):
+        tokenizer = SudachiTokenizer(normalize_text=False, sudachi_dict_type="core")
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            # fmt: off
+            [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "],
+            # fmt: on
+        )
+
+    @require_sudachi
+    def test_sudachi_tokenizer_trim_whitespace(self):
+        tokenizer = SudachiTokenizer(trim_whitespace=True, sudachi_dict_type="core")
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"],
+        )
+
+    @require_jumanpp
+    def test_pickle_jumanpp_tokenizer(self):
+        tokenizer = self.tokenizer_class(self.vocab_file, word_tokenizer_type="jumanpp")
+        self.assertIsNotNone(tokenizer)
+
+        text = "こんにちは、世界。\nこんばんは、世界。"
+        tokens = tokenizer.tokenize(text)
+        self.assertListEqual(tokens, ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
+        self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [3, 12, 10, 14, 4, 9, 12, 10, 14])
+
+        filename = os.path.join(self.tmpdirname, "tokenizer.bin")
+        with open(filename, "wb") as handle:
+            pickle.dump(tokenizer, handle)
+
+        with open(filename, "rb") as handle:
+            tokenizer_new = pickle.load(handle)
+
+        tokens_loaded = tokenizer_new.tokenize(text)
+
+        self.assertListEqual(tokens, tokens_loaded)
+
+    @require_jumanpp
+    def test_jumanpp_tokenizer(self):
+        tokenizer = JumanppTokenizer()
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            # fmt: off
+            ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"],
+            # fmt: on
+        )
+
+    @require_jumanpp
+    def test_jumanpp_tokenizer_lower(self):
+        tokenizer = JumanppTokenizer(do_lower_case=True)
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            # fmt: off
+            ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"],
+            # fmt: on
+        )
+
+    @require_jumanpp
+    def test_jumanpp_tokenizer_no_normalize(self):
+        tokenizer = JumanppTokenizer(normalize_text=False)
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            # fmt: off
+            ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"],
+            # fmt: on
+        )
+
+    @require_jumanpp
+    def test_jumanpp_tokenizer_trim_whitespace(self):
+        tokenizer = JumanppTokenizer(trim_whitespace=True)
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "),
+            ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"],
+        )
+
     def test_wordpiece_tokenizer(self):
         vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]