preparing for first release

2018-11-15 20:56:10 +01:00
parent 8513741b57
commit 1de35b624b
21 changed files with 3045 additions and 958 deletions
--- a/tests/tokenization_test.py
+++ b/tests/tokenization_test.py
@@ -34,7 +34,7 @@ class TokenizationTest(unittest.TestCase):

            vocab_file = vocab_writer.name

-        tokenizer = tokenization.FullTokenizer(vocab_file)
+        tokenizer = tokenization.BertTokenizer(vocab_file)
        os.remove(vocab_file)

        tokens = tokenizer.tokenize(u"UNwant\u00E9d,running")