fix CLIP fast tokenizer and change some properties of the slow version (#15067)

Very big changes concerning the tokenizer fast of CLIP which did not correspond to the tokenizer slow of CLIP

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
This commit is contained in:
SaulLu
2022-02-18 10:21:30 +01:00
committed by GitHub
parent 240cc6cbdc
commit e93763d420
8 changed files with 247 additions and 228 deletions

View File

@@ -105,6 +105,7 @@ _deps = [
"filelock",
"flake8>=3.8.3",
"flax>=0.3.5",
"ftfy",
"fugashi>=1.0",
"GitPython<3.1.19",
"huggingface-hub>=0.1.0,<1.0",
@@ -242,6 +243,7 @@ else:
extras["flax"] = deps_list("jax", "jaxlib", "flax", "optax")
extras["tokenizers"] = deps_list("tokenizers")
extras["ftfy"] = deps_list("ftfy")
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
extras["onnx"] = deps_list("onnxconverter-common", "tf2onnx") + extras["onnxruntime"]
extras["modelcreation"] = deps_list("cookiecutter")