add tokenizer and tests

2019-06-21 11:09:51 +02:00
parent 45709d7532
commit 32da75486b
11 changed files with 511 additions and 57 deletions
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,6 @@ boto3
 # Used for downloading models over HTTP
 requests
 # For OpenAI GPT
-regex
+regex
+# For XLNet
+sentencepiece