[tokenizers] bump to 0.21 (#34972)

bump to 0.21
2024-12-05 20:16:02 +05:30
parent 54aae121eb
commit 93f87d3cf5
2 changed files with 2 additions and 2 deletions
--- a/setup.py
+++ b/setup.py
@@ -180,7 +180,7 @@ _deps = [
    "timeout-decorator",
    "tiktoken",
    "timm<=1.0.11",
-    "tokenizers>=0.20,<0.21",
+    "tokenizers>=0.21,<0.22",
    "torch",
    "torchaudio",
    "torchvision",
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -85,7 +85,7 @@ deps = {
    "timeout-decorator": "timeout-decorator",
    "tiktoken": "tiktoken",
    "timm": "timm<=1.0.11",
-    "tokenizers": "tokenizers>=0.20,<0.21",
+    "tokenizers": "tokenizers>=0.21,<0.22",
    "torch": "torch",
    "torchaudio": "torchaudio",
    "torchvision": "torchvision",