add tokenizer and tests

2019-06-21 11:09:51 +02:00
parent 45709d7532
commit 32da75486b
11 changed files with 511 additions and 57 deletions
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,8 @@ setup(
                      'boto3',
                      'requests',
                      'tqdm',
-                      'regex'],
+                      'regex',
+                      'sentencepiece'],
    entry_points={
      'console_scripts': [
        "pytorch_pretrained_bert=pytorch_pretrained_bert.__main__:main",