Require tokenizers>=0.11.1 (#15266)

`tokenizers` version that supports the feature to choose the direction of truncation
2022-02-15 05:46:12 -05:00
parent 05a8580964
commit e1cbc073bf
2 changed files with 2 additions and 2 deletions
--- a/setup.py
+++ b/setup.py
@@ -149,7 +149,7 @@ _deps = [
    "tf2onnx",
    "timeout-decorator",
    "timm",
-    "tokenizers>=0.10.1,!=0.11.3",
+    "tokenizers>=0.11.1,!=0.11.3",
    "torch>=1.0",
    "torchaudio",
    "pyctcdecode>=0.3.0",