Use huggingface_hub helper function to split state dict (#31091)

* shard saving from hf hub * index = None * fix tests * indent
2024-06-12 14:10:32 +02:00
parent 1c73d85b86
commit 254b25abd9
4 changed files with 24 additions and 11 deletions
--- a/setup.py
+++ b/setup.py
@@ -117,7 +117,7 @@ _deps = [
    "fugashi>=1.0",
    "GitPython<3.1.19",
    "hf-doc-builder>=0.3.0",
-    "huggingface-hub>=0.23.0,<1.0",
+    "huggingface-hub>=0.23.2,<1.0",
    "importlib_metadata",
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",