Clean up hub (#18497)

* Clean up utils.hub

* Remove imports

* More fixes

* Last fix
This commit is contained in:
Sylvain Gugger
2022-08-08 08:48:10 -04:00
committed by GitHub
parent a4562552eb
commit 377cdded7a
14 changed files with 67 additions and 708 deletions

View File

@@ -24,14 +24,7 @@ from typing import Dict, Optional, Union
from huggingface_hub import HfFolder, model_info
from .utils import (
HF_MODULES_CACHE,
TRANSFORMERS_DYNAMIC_MODULE_NAME,
cached_path,
hf_bucket_url,
is_offline_mode,
logging,
)
from .utils import HF_MODULES_CACHE, TRANSFORMERS_DYNAMIC_MODULE_NAME, cached_file, is_offline_mode, logging
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
@@ -219,18 +212,15 @@ def get_cached_module_file(
# Download and cache module_file from the repo `pretrained_model_name_or_path` of grab it if it's a local file.
pretrained_model_name_or_path = str(pretrained_model_name_or_path)
if os.path.isdir(pretrained_model_name_or_path):
module_file_or_url = os.path.join(pretrained_model_name_or_path, module_file)
submodule = "local"
else:
module_file_or_url = hf_bucket_url(
pretrained_model_name_or_path, filename=module_file, revision=revision, mirror=None
)
submodule = pretrained_model_name_or_path.replace("/", os.path.sep)
try:
# Load from URL or cache if already cached
resolved_module_file = cached_path(
module_file_or_url,
resolved_module_file = cached_file(
pretrained_model_name_or_path,
module_file,
cache_dir=cache_dir,
force_download=force_download,
proxies=proxies,