Add custom tokenizer for zh and ja

2019-08-23 20:27:52 -04:00
parent 436ce07218
commit e85123d398
3 changed files with 61 additions and 22 deletions
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,4 +11,8 @@ regex
 # For XLNet
 sentencepiece
 # For XLM
-sacremoses
+sacremoses
+pythainlp
+kytea
+nltk
+jieba