From 3d1a7bf4761e332e78e07d147bacb0d26a522187 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:46:02 +0100 Subject: [PATCH] [`tokenizers`] update `tokenizers` version pin (#27494) * update `tokenizers` version pin * force tokenizers>=0.15 * use 0.14 Co-authored-by: Lysandre --------- Co-authored-by: Lysandre --- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 78763b2324..e42a29585f 100644 --- a/setup.py +++ b/setup.py @@ -175,7 +175,7 @@ _deps = [ "tf2onnx", "timeout-decorator", "timm", - "tokenizers>=0.14,<0.15", + "tokenizers>=0.14,<0.19", "torch>=1.10,!=1.12.0", "torchaudio", "torchvision", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 398e787684..736b312810 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -80,7 +80,7 @@ deps = { "tf2onnx": "tf2onnx", "timeout-decorator": "timeout-decorator", "timm": "timm", - "tokenizers": "tokenizers>=0.14,<0.15", + "tokenizers": "tokenizers>=0.14,<0.19", "torch": "torch>=1.10,!=1.12.0", "torchaudio": "torchaudio", "torchvision": "torchvision",