From 6ce6d62b6f20040129ec9831e7c4f6576402ea42 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:24:11 +0200 Subject: [PATCH] Explicit arguments in `from_pretrained` (#24306) * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix --------- Co-authored-by: ydshieh --- src/transformers/configuration_utils.py | 47 ++++++++++++++++- src/transformers/feature_extraction_utils.py | 34 +++++++++++-- .../generation/configuration_utils.py | 24 ++++++--- src/transformers/image_processing_utils.py | 34 ++++++++++++- src/transformers/modeling_flax_utils.py | 38 +++++++++----- src/transformers/modeling_tf_utils.py | 42 +++++++++++----- src/transformers/modeling_utils.py | 50 +++++++++++++------ .../models/align/configuration_align.py | 4 ++ .../models/altclip/configuration_altclip.py | 2 + .../models/blip/configuration_blip.py | 4 ++ .../models/blip_2/configuration_blip_2.py | 4 ++ .../configuration_chinese_clip.py | 4 ++ .../models/clap/configuration_clap.py | 4 ++ .../models/clip/configuration_clip.py | 4 ++ .../models/clipseg/configuration_clipseg.py | 4 ++ .../models/flava/configuration_flava.py | 8 +++ .../models/git/configuration_git.py | 2 + .../models/groupvit/configuration_groupvit.py | 4 ++ .../models/owlvit/configuration_owlvit.py | 6 +++ .../pix2struct/configuration_pix2struct.py | 4 ++ .../models/x_clip/configuration_x_clip.py | 4 ++ src/transformers/processing_utils.py | 33 +++++++++++- src/transformers/tokenization_utils_base.py | 39 +++++++++++---- 23 files changed, 336 insertions(+), 63 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index 1bcdef152a..bd1ce0e785 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -467,7 +467,43 @@ class PretrainedConfig(PushToHubMixin): ) @classmethod - def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + def _set_token_in_kwargs(self, kwargs, token=None): + """Temporary method to deal with `token` and `use_auth_token`. + + This method is to avoid apply the same changes in all model config classes that overwrite `from_pretrained`. + + Need to clean up `use_auth_token` in a follow PR. + """ + # Some model config classes like CLIP define their own `from_pretrained` without the new argument `token` yet. + if token is None: + token = kwargs.pop("token", None) + use_auth_token = kwargs.pop("use_auth_token", None) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + # change to `token` in a follow-up PR + kwargs["use_auth_token"] = token + + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ) -> "PretrainedConfig": r""" Instantiate a [`PretrainedConfig`] (or a derived class) from a pretrained model configuration. @@ -493,7 +529,7 @@ class PretrainedConfig(PushToHubMixin): proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -544,6 +580,13 @@ class PretrainedConfig(PushToHubMixin): assert config.output_attentions == True assert unused_kwargs == {"foo": False} ```""" + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + cls._set_token_in_kwargs(kwargs, token) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: logger.warning( diff --git a/src/transformers/feature_extraction_utils.py b/src/transformers/feature_extraction_utils.py index 9caf2b9f72..1eb1218d6f 100644 --- a/src/transformers/feature_extraction_utils.py +++ b/src/transformers/feature_extraction_utils.py @@ -19,6 +19,7 @@ import copy import json import os +import warnings from collections import UserDict from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union @@ -255,8 +256,15 @@ class FeatureExtractionMixin(PushToHubMixin): @classmethod def from_pretrained( - cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs - ) -> PreTrainedFeatureExtractor: + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ): r""" Instantiate a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a feature extractor, *e.g.* a derived class of [`SequenceFeatureExtractor`]. @@ -285,7 +293,7 @@ class FeatureExtractionMixin(PushToHubMixin): proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -335,6 +343,26 @@ class FeatureExtractionMixin(PushToHubMixin): assert feature_extractor.return_attention_mask is False assert unused_kwargs == {"foo": False} ```""" + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + # change to `token` in a follow-up PR + kwargs["use_auth_token"] = token + feature_extractor_dict, kwargs = cls.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs) return cls.from_dict(feature_extractor_dict, **kwargs) diff --git a/src/transformers/generation/configuration_utils.py b/src/transformers/generation/configuration_utils.py index 20caed9cc5..99426790db 100644 --- a/src/transformers/generation/configuration_utils.py +++ b/src/transformers/generation/configuration_utils.py @@ -17,6 +17,7 @@ import copy import json import os +import warnings from typing import Any, Dict, Optional, Union from .. import __version__ @@ -382,6 +383,11 @@ class GenerationConfig(PushToHubMixin): cls, pretrained_model_name: Union[str, os.PathLike], config_file_name: Optional[Union[str, os.PathLike]] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", **kwargs, ) -> "GenerationConfig": r""" @@ -410,7 +416,7 @@ class GenerationConfig(PushToHubMixin): proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -470,18 +476,24 @@ class GenerationConfig(PushToHubMixin): ```""" config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME - cache_dir = kwargs.pop("cache_dir", None) - force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) use_auth_token = kwargs.pop("use_auth_token", None) - local_files_only = kwargs.pop("local_files_only", False) - revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", "") from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) commit_hash = kwargs.pop("_commit_hash", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + user_agent = {"file_type": "config", "from_auto_class": from_auto_class} if from_pipeline is not None: user_agent["using_pipeline"] = from_pipeline @@ -509,7 +521,7 @@ class GenerationConfig(PushToHubMixin): proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + use_auth_token=token, user_agent=user_agent, revision=revision, subfolder=subfolder, diff --git a/src/transformers/image_processing_utils.py b/src/transformers/image_processing_utils.py index d51c9eebd2..a712ed490f 100644 --- a/src/transformers/image_processing_utils.py +++ b/src/transformers/image_processing_utils.py @@ -16,6 +16,7 @@ import copy import json import os +import warnings from typing import Any, Dict, Iterable, Optional, Tuple, Union import numpy as np @@ -81,7 +82,16 @@ class ImageProcessingMixin(PushToHubMixin): self._processor_class = processor_class @classmethod - def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs): + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ): r""" Instantiate a type of [`~image_processing_utils.ImageProcessingMixin`] from an image processor. @@ -109,7 +119,7 @@ class ImageProcessingMixin(PushToHubMixin): proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -162,6 +172,26 @@ class ImageProcessingMixin(PushToHubMixin): assert image_processor.do_normalize is False assert unused_kwargs == {"foo": False} ```""" + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + # change to `token` in a follow-up PR + kwargs["use_auth_token"] = token + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) return cls.from_dict(image_processor_dict, **kwargs) diff --git a/src/transformers/modeling_flax_utils.py b/src/transformers/modeling_flax_utils.py index df307d9f4b..c2732a7818 100644 --- a/src/transformers/modeling_flax_utils.py +++ b/src/transformers/modeling_flax_utils.py @@ -18,9 +18,10 @@ import gc import json import os import re +import warnings from functools import partial from pickle import UnpicklingError -from typing import Any, Dict, Set, Tuple, Union +from typing import Any, Dict, Optional, Set, Tuple, Union import flax.linen as nn import jax @@ -485,6 +486,13 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): pretrained_model_name_or_path: Union[str, os.PathLike], dtype: jnp.dtype = jnp.float32, *model_args, + config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, + ignore_mismatched_sizes: bool = False, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", **kwargs, ): r""" @@ -558,7 +566,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (i.e., do not try to download the model). - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -603,16 +611,10 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): >>> config = BertConfig.from_json_file("./pt_model/config.json") >>> model = FlaxBertModel.from_pretrained("./pt_model/pytorch_model.bin", from_pt=True, config=config) ```""" - config = kwargs.pop("config", None) - cache_dir = kwargs.pop("cache_dir", None) from_pt = kwargs.pop("from_pt", False) - ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) - force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", False) use_auth_token = kwargs.pop("use_auth_token", None) - revision = kwargs.pop("revision", None) trust_remote_code = kwargs.pop("trust_remote_code", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) @@ -620,6 +622,16 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): subfolder = kwargs.pop("subfolder", "") commit_hash = kwargs.pop("_commit_hash", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + if trust_remote_code is True: logger.warning( "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is" @@ -645,7 +657,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, _from_auto=from_auto_class, @@ -715,7 +727,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): "proxies": proxies, "resume_download": resume_download, "local_files_only": local_files_only, - "use_auth_token": use_auth_token, + "use_auth_token": token, "user_agent": user_agent, "revision": revision, "subfolder": subfolder, @@ -746,7 +758,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): has_file_kwargs = { "revision": revision, "proxies": proxies, - "use_auth_token": use_auth_token, + "use_auth_token": token, } if has_file(pretrained_model_name_or_path, WEIGHTS_NAME, **has_file_kwargs): raise EnvironmentError( @@ -797,7 +809,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + use_auth_token=token, user_agent=user_agent, revision=revision, subfolder=subfolder, @@ -986,7 +998,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin): resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, _from_auto=from_auto_class, diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py index 48876d27f7..9d0cc8687c 100644 --- a/src/transformers/modeling_tf_utils.py +++ b/src/transformers/modeling_tf_utils.py @@ -2507,7 +2507,19 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu ) @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + def from_pretrained( + cls, + pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], + *model_args, + config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, + ignore_mismatched_sizes: bool = False, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ): r""" Instantiate a pretrained TF 2.0 model from a pre-trained model configuration. @@ -2573,7 +2585,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu dictionary containing missing keys, unexpected keys and error messages. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (e.g., not try doanloading the model). - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -2629,17 +2641,11 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu >>> config = BertConfig.from_json_file("./pt_model/my_pt_model_config.json") >>> model = TFBertModel.from_pretrained("./pt_model/my_pytorch_model.bin", from_pt=True, config=config) ```""" - config = kwargs.pop("config", None) - cache_dir = kwargs.pop("cache_dir", None) from_pt = kwargs.pop("from_pt", False) - ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) - force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) output_loading_info = kwargs.pop("output_loading_info", False) - local_files_only = kwargs.pop("local_files_only", False) use_auth_token = kwargs.pop("use_auth_token", None) - revision = kwargs.pop("revision", None) trust_remote_code = kwargs.pop("trust_remote_code", None) _ = kwargs.pop("mirror", None) load_weight_prefix = kwargs.pop("load_weight_prefix", None) @@ -2649,6 +2655,16 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu commit_hash = kwargs.pop("_commit_hash", None) tf_to_pt_weight_rename = kwargs.pop("tf_to_pt_weight_rename", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + if trust_remote_code is True: logger.warning( "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is" @@ -2674,7 +2690,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=use_auth_token, revision=revision, _from_auto=from_auto_class, _from_pipeline=from_pipeline, @@ -2761,7 +2777,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu "proxies": proxies, "resume_download": resume_download, "local_files_only": local_files_only, - "use_auth_token": use_auth_token, + "use_auth_token": token, "user_agent": user_agent, "revision": revision, "subfolder": subfolder, @@ -2808,7 +2824,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu has_file_kwargs = { "revision": revision, "proxies": proxies, - "use_auth_token": use_auth_token, + "use_auth_token": token, } if has_file(pretrained_model_name_or_path, WEIGHTS_NAME, **has_file_kwargs): raise EnvironmentError( @@ -2855,7 +2871,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + use_auth_token=token, user_agent=user_agent, revision=revision, _commit_hash=commit_hash, @@ -3011,7 +3027,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, _from_auto=from_auto_class, diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 4129c52702..b3667eae0a 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1914,7 +1914,20 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix return super().float(*args) @classmethod - def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], *model_args, **kwargs): + def from_pretrained( + cls, + pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], + *model_args, + config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, + ignore_mismatched_sizes: bool = False, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + use_safetensors: bool = None, + **kwargs, + ): r""" Instantiate a pretrained pytorch model from a pre-trained model configuration. @@ -1995,7 +2008,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (i.e., do not try to download the model). - use_auth_token (`str` or `bool`, *optional*): + token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): @@ -2085,6 +2098,9 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix variant (`str`, *optional*): If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. `variant` is ignored when using `from_tf` or `from_flax`. + use_safetensors (`bool`, *optional*, defaults to `None`): + Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors` + is not installed, it will be set to `False`. kwargs (remaining dictionary of keyword arguments, *optional*): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., @@ -2142,19 +2158,13 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix Currently, it can't handle deepspeed ZeRO stage 3 and ignores loading errors """ - config = kwargs.pop("config", None) state_dict = kwargs.pop("state_dict", None) - cache_dir = kwargs.pop("cache_dir", None) from_tf = kwargs.pop("from_tf", False) from_flax = kwargs.pop("from_flax", False) - ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) - force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) output_loading_info = kwargs.pop("output_loading_info", False) - local_files_only = kwargs.pop("local_files_only", False) use_auth_token = kwargs.pop("use_auth_token", None) - revision = kwargs.pop("revision", None) trust_remote_code = kwargs.pop("trust_remote_code", None) _ = kwargs.pop("mirror", None) from_pipeline = kwargs.pop("_from_pipeline", None) @@ -2172,7 +2182,19 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix subfolder = kwargs.pop("subfolder", "") commit_hash = kwargs.pop("_commit_hash", None) variant = kwargs.pop("variant", None) - use_safetensors = kwargs.pop("use_safetensors", None if is_safetensors_available() else False) + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if use_safetensors is None and not is_safetensors_available(): + use_safetensors = False if is_bitsandbytes_available(): is_8bit_serializable = version.parse(importlib_metadata.version("bitsandbytes")) > version.parse("0.37.2") @@ -2302,7 +2324,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, _from_auto=from_auto_class, @@ -2481,7 +2503,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix "proxies": proxies, "resume_download": resume_download, "local_files_only": local_files_only, - "use_auth_token": use_auth_token, + "use_auth_token": token, "user_agent": user_agent, "revision": revision, "subfolder": subfolder, @@ -2526,7 +2548,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix has_file_kwargs = { "revision": revision, "proxies": proxies, - "use_auth_token": use_auth_token, + "use_auth_token": token, } if has_file(pretrained_model_name_or_path, TF2_WEIGHTS_NAME, **has_file_kwargs): raise EnvironmentError( @@ -2587,7 +2609,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + use_auth_token=token, user_agent=user_agent, revision=revision, subfolder=subfolder, @@ -2910,7 +2932,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix resume_download=resume_download, proxies=proxies, local_files_only=local_files_only, - use_auth_token=use_auth_token, + token=token, revision=revision, subfolder=subfolder, _from_auto=from_auto_class, diff --git a/src/transformers/models/align/configuration_align.py b/src/transformers/models/align/configuration_align.py index cb7baddfe1..787ed4b697 100644 --- a/src/transformers/models/align/configuration_align.py +++ b/src/transformers/models/align/configuration_align.py @@ -139,6 +139,8 @@ class AlignTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from AlignConfig @@ -276,6 +278,8 @@ class AlignVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from AlignConfig diff --git a/src/transformers/models/altclip/configuration_altclip.py b/src/transformers/models/altclip/configuration_altclip.py index 4ddbb5ec81..ddf22a8a52 100755 --- a/src/transformers/models/altclip/configuration_altclip.py +++ b/src/transformers/models/altclip/configuration_altclip.py @@ -228,6 +228,8 @@ class AltCLIPVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from AltCLIPConfig diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 53a7785282..a1e2ff37ef 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -161,6 +161,8 @@ class BlipTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from BlipConfig @@ -258,6 +260,8 @@ class BlipVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from BlipConfig diff --git a/src/transformers/models/blip_2/configuration_blip_2.py b/src/transformers/models/blip_2/configuration_blip_2.py index 6adf85e611..650ce264ac 100644 --- a/src/transformers/models/blip_2/configuration_blip_2.py +++ b/src/transformers/models/blip_2/configuration_blip_2.py @@ -113,6 +113,8 @@ class Blip2VisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from Blip2Config @@ -229,6 +231,8 @@ class Blip2QFormerConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the qformer config dict if we are loading from Blip2Config diff --git a/src/transformers/models/chinese_clip/configuration_chinese_clip.py b/src/transformers/models/chinese_clip/configuration_chinese_clip.py index 85dfd0e456..a1243b1852 100644 --- a/src/transformers/models/chinese_clip/configuration_chinese_clip.py +++ b/src/transformers/models/chinese_clip/configuration_chinese_clip.py @@ -144,6 +144,8 @@ class ChineseCLIPTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from ChineseCLIPConfig @@ -247,6 +249,8 @@ class ChineseCLIPVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from ChineseCLIPConfig diff --git a/src/transformers/models/clap/configuration_clap.py b/src/transformers/models/clap/configuration_clap.py index 9886c614da..32d0681377 100644 --- a/src/transformers/models/clap/configuration_clap.py +++ b/src/transformers/models/clap/configuration_clap.py @@ -144,6 +144,8 @@ class ClapTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from ClapConfig @@ -312,6 +314,8 @@ class ClapAudioConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the audio config dict if we are loading from ClapConfig diff --git a/src/transformers/models/clip/configuration_clip.py b/src/transformers/models/clip/configuration_clip.py index 2485e68933..0a3f6b9fc1 100644 --- a/src/transformers/models/clip/configuration_clip.py +++ b/src/transformers/models/clip/configuration_clip.py @@ -127,6 +127,8 @@ class CLIPTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from CLIPConfig @@ -230,6 +232,8 @@ class CLIPVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from CLIPConfig diff --git a/src/transformers/models/clipseg/configuration_clipseg.py b/src/transformers/models/clipseg/configuration_clipseg.py index 2bb7360d5c..2f05492a75 100644 --- a/src/transformers/models/clipseg/configuration_clipseg.py +++ b/src/transformers/models/clipseg/configuration_clipseg.py @@ -117,6 +117,8 @@ class CLIPSegTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from CLIPSegConfig @@ -218,6 +220,8 @@ class CLIPSegVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from CLIPSegConfig diff --git a/src/transformers/models/flava/configuration_flava.py b/src/transformers/models/flava/configuration_flava.py index f5f69ff6e5..8e899ad0f6 100644 --- a/src/transformers/models/flava/configuration_flava.py +++ b/src/transformers/models/flava/configuration_flava.py @@ -131,6 +131,8 @@ class FlavaImageConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the image config dict if we are loading from FlavaConfig @@ -258,6 +260,8 @@ class FlavaTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from FlavaConfig @@ -359,6 +363,8 @@ class FlavaMultimodalConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the multimodal config dict if we are loading from FlavaConfig @@ -442,6 +448,8 @@ class FlavaImageCodebookConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the image codebook config dict if we are loading from FlavaConfig diff --git a/src/transformers/models/git/configuration_git.py b/src/transformers/models/git/configuration_git.py index 7840ce63de..8e1a7cf2e3 100644 --- a/src/transformers/models/git/configuration_git.py +++ b/src/transformers/models/git/configuration_git.py @@ -109,6 +109,8 @@ class GitVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from GITConfig diff --git a/src/transformers/models/groupvit/configuration_groupvit.py b/src/transformers/models/groupvit/configuration_groupvit.py index 31fb19cac0..9f29b9a481 100644 --- a/src/transformers/models/groupvit/configuration_groupvit.py +++ b/src/transformers/models/groupvit/configuration_groupvit.py @@ -127,6 +127,8 @@ class GroupViTTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from GroupViTConfig @@ -250,6 +252,8 @@ class GroupViTVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from GroupViTConfig diff --git a/src/transformers/models/owlvit/configuration_owlvit.py b/src/transformers/models/owlvit/configuration_owlvit.py index 978af9dc3d..673f6d7ee6 100644 --- a/src/transformers/models/owlvit/configuration_owlvit.py +++ b/src/transformers/models/owlvit/configuration_owlvit.py @@ -127,6 +127,8 @@ class OwlViTTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from OwlViTConfig @@ -230,6 +232,8 @@ class OwlViTVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from OwlViTConfig @@ -301,6 +305,8 @@ class OwlViTConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: diff --git a/src/transformers/models/pix2struct/configuration_pix2struct.py b/src/transformers/models/pix2struct/configuration_pix2struct.py index cf1dd91b46..a86abae073 100644 --- a/src/transformers/models/pix2struct/configuration_pix2struct.py +++ b/src/transformers/models/pix2struct/configuration_pix2struct.py @@ -153,6 +153,8 @@ class Pix2StructTextConfig(PretrainedConfig): def from_pretrained( cls, pretrainehidden_size_name_or_path: Union[str, os.PathLike], **kwargs ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrainehidden_size_name_or_path, **kwargs) # get the text config dict if we are loading from Pix2StructConfig @@ -270,6 +272,8 @@ class Pix2StructVisionConfig(PretrainedConfig): def from_pretrained( cls, pretrainehidden_size_name_or_path: Union[str, os.PathLike], **kwargs ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrainehidden_size_name_or_path, **kwargs) # get the vision config dict if we are loading from Pix2StructConfig diff --git a/src/transformers/models/x_clip/configuration_x_clip.py b/src/transformers/models/x_clip/configuration_x_clip.py index ddb30d2ad2..17e8c1c230 100644 --- a/src/transformers/models/x_clip/configuration_x_clip.py +++ b/src/transformers/models/x_clip/configuration_x_clip.py @@ -118,6 +118,8 @@ class XCLIPTextConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the text config dict if we are loading from XCLIPConfig @@ -243,6 +245,8 @@ class XCLIPVisionConfig(PretrainedConfig): @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) # get the vision config dict if we are loading from XCLIPConfig diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index 09484cd132..f7f5de7d7c 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -17,7 +17,9 @@ """ import os +import warnings from pathlib import Path +from typing import Optional, Union from .dynamic_module_utils import custom_object_save from .tokenization_utils_base import PreTrainedTokenizerBase @@ -151,7 +153,16 @@ class ProcessorMixin(PushToHubMixin): ) @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ): r""" Instantiate a processor associated with a pretrained model. @@ -181,6 +192,26 @@ class ProcessorMixin(PushToHubMixin): [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. """ + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + # change to `token` in a follow-up PR + kwargs["use_auth_token"] = token + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) return cls(*args) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index eb757d98ac..bcf2d8f7ec 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1615,7 +1615,17 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): raise NotImplementedError() @classmethod - def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], *init_inputs, **kwargs): + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + *init_inputs, + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ): r""" Instantiate a [`~tokenization_utils_base.PreTrainedTokenizerBase`] (or a derived class) from a predefined tokenizer. @@ -1645,7 +1655,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. - use_auth_token (`str` or *bool*, *optional*): + token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). local_files_only (`bool`, *optional*, defaults to `False`): @@ -1692,18 +1702,29 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): # Otherwise use tokenizer.add_special_tokens({'unk_token': ''}) instead) assert tokenizer.unk_token == "" ```""" - cache_dir = kwargs.pop("cache_dir", None) - force_download = kwargs.pop("force_download", False) resume_download = kwargs.pop("resume_download", False) proxies = kwargs.pop("proxies", None) - local_files_only = kwargs.pop("local_files_only", False) use_auth_token = kwargs.pop("use_auth_token", None) - revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) commit_hash = kwargs.pop("_commit_hash", None) + use_auth_token = kwargs.pop("use_auth_token", None) + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.", FutureWarning + ) + if token is not None: + raise ValueError( + "`token` and `use_auth_token` are both specified. Please set only the argument `token`." + ) + token = use_auth_token + + if token is not None: + # change to `token` in a follow-up PR + kwargs["use_auth_token"] = token + user_agent = {"file_type": "tokenizer", "from_auto_class": from_auto_class, "is_fast": "Fast" in cls.__name__} if from_pipeline is not None: user_agent["using_pipeline"] = from_pipeline @@ -1752,7 +1773,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): force_download=force_download, resume_download=resume_download, proxies=proxies, - use_auth_token=use_auth_token, + use_auth_token=token, revision=revision, local_files_only=local_files_only, subfolder=subfolder, @@ -1789,7 +1810,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=use_auth_token, + use_auth_token=token, user_agent=user_agent, revision=revision, subfolder=subfolder, @@ -1827,7 +1848,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): pretrained_model_name_or_path, init_configuration, *init_inputs, - use_auth_token=use_auth_token, + use_auth_token=token, cache_dir=cache_dir, local_files_only=local_files_only, _commit_hash=commit_hash,