tokenizers is now a real dependency

2019-12-24 13:32:41 -05:00
parent 2818e50569
commit 734d29b03d
3 changed files with 54 additions and 66 deletions
--- a/setup.py
+++ b/setup.py
@@ -86,6 +86,7 @@ setup(
    packages=find_packages("src"),
    install_requires=[
        "numpy",
+        "tokenizers",
        # accessing files from S3 directly
        "boto3",
        # filesystem locks e.g. to prevent parallel downloads