Update tokenizers to 0.7.0-rc5 (#3705)

2020-04-10 14:23:49 -04:00
parent 551b450527
commit b7cf9f43d2
4 changed files with 6 additions and 8 deletions
--- a/setup.py
+++ b/setup.py
@@ -96,7 +96,7 @@ setup(
    packages=find_packages("src"),
    install_requires=[
        "numpy",
-        "tokenizers == 0.7.0rc3",
+        "tokenizers == 0.7.0rc5",
        # dataclasses for Python versions that don't have it
        "dataclasses;python_version<'3.7'",
        # accessing files from S3 directly