Use commit hash to look in cache instead of calling head (#18534)
* Use commit hash to look in cache instead of calling head * Add tests * Add attr for local configs too * Stupid typos * Fix tests * Update src/transformers/utils/hub.py Co-authored-by: Julien Chaumond <julien@huggingface.co> * Address Julien's comments Co-authored-by: Julien Chaumond <julien@huggingface.co>
This commit is contained in:
@@ -31,6 +31,7 @@ from pathlib import Path
|
||||
from typing import Iterator, List, Union
|
||||
from unittest import mock
|
||||
|
||||
import huggingface_hub
|
||||
from transformers import logging as transformers_logging
|
||||
|
||||
from .deepspeed import is_deepspeed_available
|
||||
@@ -1588,3 +1589,30 @@ def run_command(command: List[str], return_stdout=False):
|
||||
raise SubprocessCallException(
|
||||
f"Command `{' '.join(command)}` failed with the following error:\n\n{e.output.decode()}"
|
||||
) from e
|
||||
|
||||
|
||||
class RequestCounter:
|
||||
"""
|
||||
Helper class that will count all requests made online.
|
||||
"""
|
||||
|
||||
def __enter__(self):
|
||||
self.head_request_count = 0
|
||||
self.get_request_count = 0
|
||||
self.other_request_count = 0
|
||||
self.old_request = huggingface_hub.file_download.requests.request
|
||||
huggingface_hub.file_download.requests.request = self.new_request
|
||||
return self
|
||||
|
||||
def __exit__(self, *args, **kwargs):
|
||||
huggingface_hub.file_download.requests.request = self.old_request
|
||||
|
||||
def new_request(self, method, **kwargs):
|
||||
if method == "GET":
|
||||
self.get_request_count += 1
|
||||
elif method == "HEAD":
|
||||
self.head_request_count += 1
|
||||
else:
|
||||
self.other_request_count += 1
|
||||
|
||||
return self.old_request(method=method, **kwargs)
|
||||
|
||||
Reference in New Issue
Block a user