Make bert_japanese and cpm independent of their inherited modules (#19431)
* Make cpm tokenization independent of xlnet * Make bert japanese tokenization independent of bert
This commit is contained in:
@@ -19,10 +19,10 @@ import pickle
|
||||
import unittest
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
from transformers.models.bert.tokenization_bert import BertTokenizer
|
||||
from transformers.models.bert_japanese.tokenization_bert_japanese import (
|
||||
VOCAB_FILES_NAMES,
|
||||
BertJapaneseTokenizer,
|
||||
BertTokenizer,
|
||||
CharacterTokenizer,
|
||||
JumanppTokenizer,
|
||||
MecabTokenizer,
|
||||
|
||||
Reference in New Issue
Block a user