[tokenizers] bump to 0.21 (#34972)

bump to 0.21
2024-12-05 20:16:02 +05:30
parent 54aae121eb
commit 93f87d3cf5
2 changed files with 2 additions and 2 deletions
--- a/setup.py
+++ b/setup.py
@@ -180,7 +180,7 @@ _deps = [
    "timeout-decorator",
    "tiktoken",
    "timm<=1.0.11",
-    "tokenizers>=0.20,<0.21",
+    "tokenizers>=0.21,<0.22",
    "torch",
    "torchaudio",
    "torchvision",