Update for changes in tokenizers API

2019-12-26 14:37:55 -05:00
parent 734d29b03d
commit 1f82a5d910
4 changed files with 20 additions and 16 deletions
--- a/setup.py
+++ b/setup.py
@@ -86,7 +86,7 @@ setup(
    packages=find_packages("src"),
    install_requires=[
        "numpy",
-        "tokenizers",
+        "tokenizers == 0.0.10",
        # accessing files from S3 directly
        "boto3",
        # filesystem locks e.g. to prevent parallel downloads