fix CLIP fast tokenizer and change some properties of the slow version (#15067)
Very big changes concerning the tokenizer fast of CLIP which did not correspond to the tokenizer slow of CLIP Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
This commit is contained in:
2
setup.py
2
setup.py
@@ -105,6 +105,7 @@ _deps = [
|
||||
"filelock",
|
||||
"flake8>=3.8.3",
|
||||
"flax>=0.3.5",
|
||||
"ftfy",
|
||||
"fugashi>=1.0",
|
||||
"GitPython<3.1.19",
|
||||
"huggingface-hub>=0.1.0,<1.0",
|
||||
@@ -242,6 +243,7 @@ else:
|
||||
extras["flax"] = deps_list("jax", "jaxlib", "flax", "optax")
|
||||
|
||||
extras["tokenizers"] = deps_list("tokenizers")
|
||||
extras["ftfy"] = deps_list("ftfy")
|
||||
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
|
||||
extras["onnx"] = deps_list("onnxconverter-common", "tf2onnx") + extras["onnxruntime"]
|
||||
extras["modelcreation"] = deps_list("cookiecutter")
|
||||
|
||||
Reference in New Issue
Block a user