[tokenizers] update tokenizers version pin (#27494)

* update `tokenizers` version pin * force tokenizers>=0.15 * use 0.14 Co-authored-by: Lysandre <lysandre@huggingface.co> --------- Co-authored-by: Lysandre <lysandre@huggingface.co>
2023-11-15 10:46:02 +01:00
parent 64e21ca2a4
commit 3d1a7bf476
2 changed files with 2 additions and 2 deletions
--- a/setup.py
+++ b/setup.py
@@ -175,7 +175,7 @@ _deps = [
    "tf2onnx",
    "timeout-decorator",
    "timm",
-    "tokenizers>=0.14,<0.15",
+    "tokenizers>=0.14,<0.19",
    "torch>=1.10,!=1.12.0",
    "torchaudio",
    "torchvision",