From b97cab7e6d3934c566e6cc8daf3e2f76d92397d5 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:20:54 +0100 Subject: [PATCH] Remove-auth-token (#27060) * don't use `use_auth_token`internally * let's use token everywhere * fixup --- docs/source/en/hpo_train.md | 2 +- docs/source/en/model_doc/nllb.md | 4 ++-- docs/source/ja/hpo_train.md | 2 +- docs/source/ko/hpo_train.md | 2 +- .../research_projects/jax-projects/README.md | 2 +- .../layoutlmv3/run_funsd_cord.py | 10 +++++----- .../research_projects/mlm_wwm/run_mlm_wwm.py | 2 +- .../quantization-qdqbert/run_quant_qa.py | 6 +++--- .../robust-speech-event/eval.py | 2 +- .../run_speech_recognition_ctc_bnb.py | 12 ++++++------ .../run_speech_recognition_ctc_streaming.py | 12 ++++++------ .../tapex/run_tabfact_with_tapex.py | 6 +++--- .../tapex/run_wikisql_with_tapex.py | 6 +++--- .../tapex/run_wikitablequestions_with_tapex.py | 6 +++--- .../research_projects/xtreme-s/run_xtreme_s.py | 18 +++++++++--------- .../generation/configuration_utils.py | 2 +- src/transformers/integrations/peft.py | 4 ++-- .../models/bark/processing_bark.py | 4 ++-- src/transformers/tools/base.py | 6 +++--- tests/models/auto/test_processor_auto.py | 6 ++---- .../pytorch/run_glue_model_parallelism.py | 6 +++--- tests/test_configuration_utils.py | 8 +++----- tests/test_feature_extraction_utils.py | 10 +++++----- tests/test_image_processing_utils.py | 10 +++++----- tests/test_modeling_flax_utils.py | 8 ++++---- tests/test_modeling_tf_utils.py | 10 ++++------ tests/test_modeling_utils.py | 12 +++++------- tests/test_tokenization_utils.py | 12 ++++++------ tests/utils/test_hub_utils.py | 4 ++-- 29 files changed, 93 insertions(+), 101 deletions(-) diff --git a/docs/source/en/hpo_train.md b/docs/source/en/hpo_train.md index 882193d9e8..c516c501f8 100644 --- a/docs/source/en/hpo_train.md +++ b/docs/source/en/hpo_train.md @@ -99,7 +99,7 @@ Define a `model_init` function and pass it to the [`Trainer`], as an example: ... config=config, ... cache_dir=model_args.cache_dir, ... revision=model_args.model_revision, -... use_auth_token=True if model_args.use_auth_token else None, +... token=True if model_args.use_auth_token else None, ... ) ``` diff --git a/docs/source/en/model_doc/nllb.md b/docs/source/en/model_doc/nllb.md index b0dffa185e..3f272129d2 100644 --- a/docs/source/en/model_doc/nllb.md +++ b/docs/source/en/model_doc/nllb.md @@ -118,9 +118,9 @@ See example below for a translation from romanian to german: >>> from transformers import AutoModelForSeq2SeqLM, AutoTokenizer >>> tokenizer = AutoTokenizer.from_pretrained( -... "facebook/nllb-200-distilled-600M", use_auth_token=True, src_lang="ron_Latn" +... "facebook/nllb-200-distilled-600M", token=True, src_lang="ron_Latn" ... ) ->>> model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M", use_auth_token=True) +>>> model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M", token=True) >>> article = "Şeful ONU spune că nu există o soluţie militară în Siria" >>> inputs = tokenizer(article, return_tensors="pt") diff --git a/docs/source/ja/hpo_train.md b/docs/source/ja/hpo_train.md index 46a22550bb..85da3616f8 100644 --- a/docs/source/ja/hpo_train.md +++ b/docs/source/ja/hpo_train.md @@ -105,7 +105,7 @@ Wandbについては、[object_parameter](https://docs.wandb.ai/guides/sweeps/co ... config=config, ... cache_dir=model_args.cache_dir, ... revision=model_args.model_revision, -... use_auth_token=True if model_args.use_auth_token else None, +... token=True if model_args.use_auth_token else None, ... ) ``` diff --git a/docs/source/ko/hpo_train.md b/docs/source/ko/hpo_train.md index c7b2530693..58bacd55ff 100644 --- a/docs/source/ko/hpo_train.md +++ b/docs/source/ko/hpo_train.md @@ -87,7 +87,7 @@ wandb의 경우, 해당 [object_parameter](https://docs.wandb.ai/guides/sweeps/c ... config=config, ... cache_dir=model_args.cache_dir, ... revision=model_args.model_revision, -... use_auth_token=True if model_args.use_auth_token else None, +... token=True if model_args.use_auth_token else None, ... ) ``` diff --git a/examples/research_projects/jax-projects/README.md b/examples/research_projects/jax-projects/README.md index 66bb6c61a3..fc5f096955 100644 --- a/examples/research_projects/jax-projects/README.md +++ b/examples/research_projects/jax-projects/README.md @@ -1117,7 +1117,7 @@ params = model.init(key2, x) bytes_output = serialization.to_bytes(params) -repo = Repository("flax-model", clone_from="flax-community/flax-model-dummy", use_auth_token=True) +repo = Repository("flax-model", clone_from="flax-community/flax-model-dummy", token=True) with repo.commit("My cool Flax model :)"): with open("flax_model.msgpack", "wb") as f: f.write(bytes_output) diff --git a/examples/research_projects/layoutlmv3/run_funsd_cord.py b/examples/research_projects/layoutlmv3/run_funsd_cord.py index e826fd9974..ad83fbdef9 100644 --- a/examples/research_projects/layoutlmv3/run_funsd_cord.py +++ b/examples/research_projects/layoutlmv3/run_funsd_cord.py @@ -250,7 +250,7 @@ def main(): "nielsr/funsd-layoutlmv3", data_args.dataset_config_name, cache_dir=model_args.cache_dir, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) elif data_args.dataset_name == "cord": # Downloading and loading a dataset from the hub. @@ -258,7 +258,7 @@ def main(): "nielsr/cord-layoutlmv3", data_args.dataset_config_name, cache_dir=model_args.cache_dir, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) else: raise ValueError("This script only supports either FUNSD or CORD out-of-the-box.") @@ -313,7 +313,7 @@ def main(): finetuning_task=data_args.task_name, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) processor = AutoProcessor.from_pretrained( @@ -321,7 +321,7 @@ def main(): cache_dir=model_args.cache_dir, use_fast=True, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, add_prefix_space=True, apply_ocr=False, ) @@ -332,7 +332,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # Set the correspondences label/ID inside the model config diff --git a/examples/research_projects/mlm_wwm/run_mlm_wwm.py b/examples/research_projects/mlm_wwm/run_mlm_wwm.py index 84272421b9..d22b2db7dc 100644 --- a/examples/research_projects/mlm_wwm/run_mlm_wwm.py +++ b/examples/research_projects/mlm_wwm/run_mlm_wwm.py @@ -325,7 +325,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) else: logger.info("Training new model from scratch") diff --git a/examples/research_projects/quantization-qdqbert/run_quant_qa.py b/examples/research_projects/quantization-qdqbert/run_quant_qa.py index bb887955eb..fac834ef70 100755 --- a/examples/research_projects/quantization-qdqbert/run_quant_qa.py +++ b/examples/research_projects/quantization-qdqbert/run_quant_qa.py @@ -322,14 +322,14 @@ def main(): model_args.config_name if model_args.config_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) tokenizer = AutoTokenizer.from_pretrained( model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=True, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) model = QDQBertForQuestionAnswering.from_pretrained( model_args.model_name_or_path, @@ -337,7 +337,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # Tokenizer check: this script requires a fast tokenizer. diff --git a/examples/research_projects/robust-speech-event/eval.py b/examples/research_projects/robust-speech-event/eval.py index a8acca1825..b6c89a6d49 100755 --- a/examples/research_projects/robust-speech-event/eval.py +++ b/examples/research_projects/robust-speech-event/eval.py @@ -65,7 +65,7 @@ def normalize_text(text: str) -> str: def main(args): # load dataset - dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True) + dataset = load_dataset(args.dataset, args.config, split=args.split, token=True) # for testing: only process the first two examples as a test # dataset = dataset.select(range(10)) diff --git a/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_bnb.py b/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_bnb.py index 7a137eb5fe..ebf33eb01d 100755 --- a/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_bnb.py +++ b/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_bnb.py @@ -418,7 +418,7 @@ def main(): data_args.dataset_name, data_args.dataset_config_name, split=data_args.train_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) if data_args.audio_column_name not in raw_datasets["train"].column_names: @@ -443,7 +443,7 @@ def main(): data_args.dataset_name, data_args.dataset_config_name, split=data_args.eval_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) if data_args.max_eval_samples is not None: @@ -481,7 +481,7 @@ def main(): # the tokenizer # load config config = AutoConfig.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token ) # 4. Next, if no tokenizer file is defined, @@ -532,11 +532,11 @@ def main(): # load feature_extractor and tokenizer tokenizer = AutoTokenizer.from_pretrained( tokenizer_name_or_path, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, **tokenizer_kwargs, ) feature_extractor = AutoFeatureExtractor.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token ) # adapt config @@ -564,7 +564,7 @@ def main(): model_args.model_name_or_path, cache_dir=model_args.cache_dir, config=config, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) # freeze encoder diff --git a/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_streaming.py b/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_streaming.py index e3800142e3..8a8eda851b 100644 --- a/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_streaming.py +++ b/examples/research_projects/robust-speech-event/run_speech_recognition_ctc_streaming.py @@ -395,7 +395,7 @@ def main(): # so we just need to set the correct target sampling rate and normalize the input # via the `feature_extractor` feature_extractor = AutoFeatureExtractor.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token ) if training_args.do_train: @@ -403,7 +403,7 @@ def main(): path=data_args.dataset_name, name=data_args.dataset_config_name, split=data_args.train_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, streaming=True, sampling_rate=feature_extractor.sampling_rate, ) @@ -431,7 +431,7 @@ def main(): path=data_args.dataset_name, name=data_args.dataset_config_name, split=data_args.eval_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, streaming=True, sampling_rate=feature_extractor.sampling_rate, ) @@ -465,7 +465,7 @@ def main(): # 3. Next, let's load the config as we might need it to create # the tokenizer config = AutoConfig.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token ) # 4. Now we can instantiate the tokenizer and model @@ -481,7 +481,7 @@ def main(): tokenizer = AutoTokenizer.from_pretrained( tokenizer_name_or_path, config=config, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) # adapt config @@ -509,7 +509,7 @@ def main(): model_args.model_name_or_path, cache_dir=model_args.cache_dir, config=config, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) # freeze encoder diff --git a/examples/research_projects/tapex/run_tabfact_with_tapex.py b/examples/research_projects/tapex/run_tabfact_with_tapex.py index 17222b8c45..2bef4a371e 100644 --- a/examples/research_projects/tapex/run_tabfact_with_tapex.py +++ b/examples/research_projects/tapex/run_tabfact_with_tapex.py @@ -292,7 +292,7 @@ def main(): num_labels=num_labels, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # load tapex tokenizer tokenizer = TapexTokenizer.from_pretrained( @@ -300,7 +300,7 @@ def main(): cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, add_prefix_space=True, ) model = BartForSequenceClassification.from_pretrained( @@ -309,7 +309,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # Padding strategy diff --git a/examples/research_projects/tapex/run_wikisql_with_tapex.py b/examples/research_projects/tapex/run_wikisql_with_tapex.py index 3e72a5ad53..821b283d9f 100644 --- a/examples/research_projects/tapex/run_wikisql_with_tapex.py +++ b/examples/research_projects/tapex/run_wikisql_with_tapex.py @@ -329,7 +329,7 @@ def main(): model_args.config_name if model_args.config_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # IMPORTANT: the initial BART model's decoding is penalized by no_repeat_ngram_size, and thus @@ -344,7 +344,7 @@ def main(): cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, add_prefix_space=True, ) @@ -355,7 +355,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) if model.config.decoder_start_token_id is None: diff --git a/examples/research_projects/tapex/run_wikitablequestions_with_tapex.py b/examples/research_projects/tapex/run_wikitablequestions_with_tapex.py index bec81a9889..f874eebb34 100644 --- a/examples/research_projects/tapex/run_wikitablequestions_with_tapex.py +++ b/examples/research_projects/tapex/run_wikitablequestions_with_tapex.py @@ -327,7 +327,7 @@ def main(): model_args.config_name if model_args.config_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # IMPORTANT: the initial BART model's decoding is penalized by no_repeat_ngram_size, and thus @@ -342,7 +342,7 @@ def main(): cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, add_prefix_space=True, ) @@ -353,7 +353,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) if model.config.decoder_start_token_id is None: diff --git a/examples/research_projects/xtreme-s/run_xtreme_s.py b/examples/research_projects/xtreme-s/run_xtreme_s.py index 5db30c7e97..e01ccbf448 100644 --- a/examples/research_projects/xtreme-s/run_xtreme_s.py +++ b/examples/research_projects/xtreme-s/run_xtreme_s.py @@ -502,7 +502,7 @@ def main(): data_args.dataset_name, config_name, split=data_args.train_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, cache_dir=model_args.cache_dir, ) @@ -528,7 +528,7 @@ def main(): data_args.dataset_name, config_name, split=data_args.eval_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, cache_dir=model_args.cache_dir, ) @@ -540,7 +540,7 @@ def main(): data_args.dataset_name, config_name, split=data_args.predict_split_name, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, cache_dir=model_args.cache_dir, ) @@ -595,7 +595,7 @@ def main(): # 3. Next, let's load the config as we might need it to create # the tokenizer config = AutoConfig.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token ) if is_text_target: @@ -651,11 +651,11 @@ def main(): if is_text_target: tokenizer = AutoTokenizer.from_pretrained( tokenizer_name_or_path, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, **tokenizer_kwargs, ) feature_extractor = AutoFeatureExtractor.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token ) # adapt config @@ -694,14 +694,14 @@ def main(): model_args.model_name_or_path, cache_dir=model_args.cache_dir, config=config, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) elif config.is_encoder_decoder: model = AutoModelForSpeechSeq2Seq.from_pretrained( model_args.model_name_or_path, cache_dir=model_args.cache_dir, config=config, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) if model.config.decoder_start_token_id is None: raise ValueError("Make sure that `config.decoder_start_token_id` is correctly defined") @@ -710,7 +710,7 @@ def main(): model_args.model_name_or_path, cache_dir=model_args.cache_dir, config=config, - use_auth_token=data_args.use_auth_token, + token=data_args.use_auth_token, ) # freeze encoder diff --git a/src/transformers/generation/configuration_utils.py b/src/transformers/generation/configuration_utils.py index ef3a6fba1b..04aa48364f 100644 --- a/src/transformers/generation/configuration_utils.py +++ b/src/transformers/generation/configuration_utils.py @@ -716,7 +716,7 @@ class GenerationConfig(PushToHubMixin): proxies=proxies, resume_download=resume_download, local_files_only=local_files_only, - use_auth_token=token, + token=token, user_agent=user_agent, revision=revision, subfolder=subfolder, diff --git a/src/transformers/integrations/peft.py b/src/transformers/integrations/peft.py index de68e01c5f..8be09e8631 100644 --- a/src/transformers/integrations/peft.py +++ b/src/transformers/integrations/peft.py @@ -179,7 +179,7 @@ class PeftAdapterMixin: peft_config = PeftConfig.from_pretrained( peft_model_id, - use_auth_token=token, + token=token, **adapter_kwargs, ) @@ -190,7 +190,7 @@ class PeftAdapterMixin: self._hf_peft_config_loaded = True if peft_model_id is not None: - adapter_state_dict = load_peft_weights(peft_model_id, use_auth_token=token, **adapter_kwargs) + adapter_state_dict = load_peft_weights(peft_model_id, token=token, **adapter_kwargs) # We need to pre-process the state dict to remove unneeded prefixes - for backward compatibility processed_adapter_state_dict = {} diff --git a/src/transformers/models/bark/processing_bark.py b/src/transformers/models/bark/processing_bark.py index 2b38132759..ef8ed03224 100644 --- a/src/transformers/models/bark/processing_bark.py +++ b/src/transformers/models/bark/processing_bark.py @@ -94,7 +94,7 @@ class BarkProcessor(ProcessorMixin): proxies=kwargs.pop("proxies", None), resume_download=kwargs.pop("resume_download", False), local_files_only=kwargs.pop("local_files_only", False), - use_auth_token=kwargs.pop("use_auth_token", None), + token=kwargs.pop("use_auth_token", None), revision=kwargs.pop("revision", None), ) if speaker_embeddings_path is None: @@ -190,7 +190,7 @@ class BarkProcessor(ProcessorMixin): proxies=kwargs.pop("proxies", None), resume_download=kwargs.pop("resume_download", False), local_files_only=kwargs.pop("local_files_only", False), - use_auth_token=kwargs.pop("use_auth_token", None), + token=kwargs.pop("use_auth_token", None), revision=kwargs.pop("revision", None), ) if path is None: diff --git a/src/transformers/tools/base.py b/src/transformers/tools/base.py index c781a0b270..42027948a6 100644 --- a/src/transformers/tools/base.py +++ b/src/transformers/tools/base.py @@ -226,7 +226,7 @@ class Tool: resolved_config_file = cached_file( repo_id, TOOL_CONFIG_FILE, - use_auth_token=token, + token=token, **hub_kwargs, _raise_exceptions_for_missing_entries=False, _raise_exceptions_for_connection_errors=False, @@ -236,7 +236,7 @@ class Tool: resolved_config_file = cached_file( repo_id, CONFIG_NAME, - use_auth_token=token, + token=token, **hub_kwargs, _raise_exceptions_for_missing_entries=False, _raise_exceptions_for_connection_errors=False, @@ -259,7 +259,7 @@ class Tool: custom_tool = config tool_class = custom_tool["tool_class"] - tool_class = get_class_from_dynamic_module(tool_class, repo_id, use_auth_token=token, **hub_kwargs) + tool_class = get_class_from_dynamic_module(tool_class, repo_id, token=token, **hub_kwargs) if len(tool_class.name) == 0: tool_class.name = custom_tool["name"] diff --git a/tests/models/auto/test_processor_auto.py b/tests/models/auto/test_processor_auto.py index a4f3714268..bf4a92475d 100644 --- a/tests/models/auto/test_processor_auto.py +++ b/tests/models/auto/test_processor_auto.py @@ -308,9 +308,7 @@ class ProcessorPushToHubTester(unittest.TestCase): def test_push_to_hub(self): processor = Wav2Vec2Processor.from_pretrained(SAMPLE_PROCESSOR_CONFIG_DIR) with tempfile.TemporaryDirectory() as tmp_dir: - processor.save_pretrained( - os.path.join(tmp_dir, "test-processor"), push_to_hub=True, use_auth_token=self._token - ) + processor.save_pretrained(os.path.join(tmp_dir, "test-processor"), push_to_hub=True, token=self._token) new_processor = Wav2Vec2Processor.from_pretrained(f"{USER}/test-processor") for k, v in processor.feature_extractor.__dict__.items(): @@ -324,7 +322,7 @@ class ProcessorPushToHubTester(unittest.TestCase): processor.save_pretrained( os.path.join(tmp_dir, "test-processor-org"), push_to_hub=True, - use_auth_token=self._token, + token=self._token, organization="valid_org", ) diff --git a/tests/sagemaker/scripts/pytorch/run_glue_model_parallelism.py b/tests/sagemaker/scripts/pytorch/run_glue_model_parallelism.py index 32e28e4697..c38ee542e6 100644 --- a/tests/sagemaker/scripts/pytorch/run_glue_model_parallelism.py +++ b/tests/sagemaker/scripts/pytorch/run_glue_model_parallelism.py @@ -314,14 +314,14 @@ def main(): finetuning_task=data_args.task_name, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) tokenizer = AutoTokenizer.from_pretrained( model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) model = AutoModelForSequenceClassification.from_pretrained( model_args.model_name_or_path, @@ -329,7 +329,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - use_auth_token=True if model_args.use_auth_token else None, + token=True if model_args.use_auth_token else None, ) # Preprocessing the datasets diff --git a/tests/test_configuration_utils.py b/tests/test_configuration_utils.py index 1b8136bfbb..a6e9e6b039 100644 --- a/tests/test_configuration_utils.py +++ b/tests/test_configuration_utils.py @@ -142,7 +142,7 @@ class ConfigPushToHubTester(unittest.TestCase): config = BertConfig( vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 ) - config.push_to_hub("valid_org/test-config-org", use_auth_token=self._token) + config.push_to_hub("valid_org/test-config-org", token=self._token) new_config = BertConfig.from_pretrained("valid_org/test-config-org") for k, v in config.to_dict().items(): @@ -154,9 +154,7 @@ class ConfigPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - config.save_pretrained( - tmp_dir, repo_id="valid_org/test-config-org", push_to_hub=True, use_auth_token=self._token - ) + config.save_pretrained(tmp_dir, repo_id="valid_org/test-config-org", push_to_hub=True, token=self._token) new_config = BertConfig.from_pretrained("valid_org/test-config-org") for k, v in config.to_dict().items(): @@ -167,7 +165,7 @@ class ConfigPushToHubTester(unittest.TestCase): CustomConfig.register_for_auto_class() config = CustomConfig(attribute=42) - config.push_to_hub("test-dynamic-config", use_auth_token=self._token) + config.push_to_hub("test-dynamic-config", token=self._token) # This has added the proper auto_map field to the config self.assertDictEqual(config.auto_map, {"AutoConfig": "custom_configuration.CustomConfig"}) diff --git a/tests/test_feature_extraction_utils.py b/tests/test_feature_extraction_utils.py index b17c48ff12..7467cac139 100644 --- a/tests/test_feature_extraction_utils.py +++ b/tests/test_feature_extraction_utils.py @@ -85,7 +85,7 @@ class FeatureExtractorPushToHubTester(unittest.TestCase): def test_push_to_hub(self): feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) - feature_extractor.push_to_hub("test-feature-extractor", use_auth_token=self._token) + feature_extractor.push_to_hub("test-feature-extractor", token=self._token) new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor") for k, v in feature_extractor.__dict__.items(): @@ -97,7 +97,7 @@ class FeatureExtractorPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: feature_extractor.save_pretrained( - tmp_dir, repo_id="test-feature-extractor", push_to_hub=True, use_auth_token=self._token + tmp_dir, repo_id="test-feature-extractor", push_to_hub=True, token=self._token ) new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor") @@ -106,7 +106,7 @@ class FeatureExtractorPushToHubTester(unittest.TestCase): def test_push_to_hub_in_organization(self): feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) - feature_extractor.push_to_hub("valid_org/test-feature-extractor", use_auth_token=self._token) + feature_extractor.push_to_hub("valid_org/test-feature-extractor", token=self._token) new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("valid_org/test-feature-extractor") for k, v in feature_extractor.__dict__.items(): @@ -118,7 +118,7 @@ class FeatureExtractorPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: feature_extractor.save_pretrained( - tmp_dir, repo_id="valid_org/test-feature-extractor-org", push_to_hub=True, use_auth_token=self._token + tmp_dir, repo_id="valid_org/test-feature-extractor-org", push_to_hub=True, token=self._token ) new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("valid_org/test-feature-extractor-org") @@ -129,7 +129,7 @@ class FeatureExtractorPushToHubTester(unittest.TestCase): CustomFeatureExtractor.register_for_auto_class() feature_extractor = CustomFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) - feature_extractor.push_to_hub("test-dynamic-feature-extractor", use_auth_token=self._token) + feature_extractor.push_to_hub("test-dynamic-feature-extractor", token=self._token) # This has added the proper auto_map field to the config self.assertDictEqual( diff --git a/tests/test_image_processing_utils.py b/tests/test_image_processing_utils.py index 5211a541d3..3be8ee3c26 100644 --- a/tests/test_image_processing_utils.py +++ b/tests/test_image_processing_utils.py @@ -95,7 +95,7 @@ class ImageProcessorPushToHubTester(unittest.TestCase): def test_push_to_hub(self): image_processor = ViTImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) - image_processor.push_to_hub("test-image-processor", use_auth_token=self._token) + image_processor.push_to_hub("test-image-processor", token=self._token) new_image_processor = ViTImageProcessor.from_pretrained(f"{USER}/test-image-processor") for k, v in image_processor.__dict__.items(): @@ -107,7 +107,7 @@ class ImageProcessorPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: image_processor.save_pretrained( - tmp_dir, repo_id="test-image-processor", push_to_hub=True, use_auth_token=self._token + tmp_dir, repo_id="test-image-processor", push_to_hub=True, token=self._token ) new_image_processor = ViTImageProcessor.from_pretrained(f"{USER}/test-image-processor") @@ -116,7 +116,7 @@ class ImageProcessorPushToHubTester(unittest.TestCase): def test_push_to_hub_in_organization(self): image_processor = ViTImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) - image_processor.push_to_hub("valid_org/test-image-processor", use_auth_token=self._token) + image_processor.push_to_hub("valid_org/test-image-processor", token=self._token) new_image_processor = ViTImageProcessor.from_pretrained("valid_org/test-image-processor") for k, v in image_processor.__dict__.items(): @@ -128,7 +128,7 @@ class ImageProcessorPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: image_processor.save_pretrained( - tmp_dir, repo_id="valid_org/test-image-processor-org", push_to_hub=True, use_auth_token=self._token + tmp_dir, repo_id="valid_org/test-image-processor-org", push_to_hub=True, token=self._token ) new_image_processor = ViTImageProcessor.from_pretrained("valid_org/test-image-processor-org") @@ -139,7 +139,7 @@ class ImageProcessorPushToHubTester(unittest.TestCase): CustomImageProcessor.register_for_auto_class() image_processor = CustomImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) - image_processor.push_to_hub("test-dynamic-image-processor", use_auth_token=self._token) + image_processor.push_to_hub("test-dynamic-image-processor", token=self._token) # This has added the proper auto_map field to the config self.assertDictEqual( diff --git a/tests/test_modeling_flax_utils.py b/tests/test_modeling_flax_utils.py index 06ed30f8af..e0e6c873c6 100644 --- a/tests/test_modeling_flax_utils.py +++ b/tests/test_modeling_flax_utils.py @@ -60,7 +60,7 @@ class FlaxModelPushToHubTester(unittest.TestCase): vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 ) model = FlaxBertModel(config) - model.push_to_hub("test-model-flax", use_auth_token=self._token) + model.push_to_hub("test-model-flax", token=self._token) new_model = FlaxBertModel.from_pretrained(f"{USER}/test-model-flax") @@ -76,7 +76,7 @@ class FlaxModelPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, repo_id="test-model-flax", push_to_hub=True, use_auth_token=self._token) + model.save_pretrained(tmp_dir, repo_id="test-model-flax", push_to_hub=True, token=self._token) new_model = FlaxBertModel.from_pretrained(f"{USER}/test-model-flax") @@ -92,7 +92,7 @@ class FlaxModelPushToHubTester(unittest.TestCase): vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 ) model = FlaxBertModel(config) - model.push_to_hub("valid_org/test-model-flax-org", use_auth_token=self._token) + model.push_to_hub("valid_org/test-model-flax-org", token=self._token) new_model = FlaxBertModel.from_pretrained("valid_org/test-model-flax-org") @@ -109,7 +109,7 @@ class FlaxModelPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: model.save_pretrained( - tmp_dir, repo_id="valid_org/test-model-flax-org", push_to_hub=True, use_auth_token=self._token + tmp_dir, repo_id="valid_org/test-model-flax-org", push_to_hub=True, token=self._token ) new_model = FlaxBertModel.from_pretrained("valid_org/test-model-flax-org") diff --git a/tests/test_modeling_tf_utils.py b/tests/test_modeling_tf_utils.py index 6d0ed86407..130f920f71 100644 --- a/tests/test_modeling_tf_utils.py +++ b/tests/test_modeling_tf_utils.py @@ -572,7 +572,7 @@ class TFModelPushToHubTester(unittest.TestCase): logging.set_verbosity_info() logger = logging.get_logger("transformers.utils.hub") with CaptureLogger(logger) as cl: - model.push_to_hub("test-model-tf", use_auth_token=self._token) + model.push_to_hub("test-model-tf", token=self._token) logging.set_verbosity_warning() # Check the model card was created and uploaded. self.assertIn("Uploading the following files to __DUMMY_TRANSFORMERS_USER__/test-model-tf", cl.out) @@ -590,7 +590,7 @@ class TFModelPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, repo_id="test-model-tf", push_to_hub=True, use_auth_token=self._token) + model.save_pretrained(tmp_dir, repo_id="test-model-tf", push_to_hub=True, token=self._token) new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") models_equal = True @@ -638,7 +638,7 @@ class TFModelPushToHubTester(unittest.TestCase): # Make sure model is properly initialized model.build() - model.push_to_hub("valid_org/test-model-tf-org", use_auth_token=self._token) + model.push_to_hub("valid_org/test-model-tf-org", token=self._token) new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") models_equal = True @@ -653,9 +653,7 @@ class TFModelPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained( - tmp_dir, push_to_hub=True, use_auth_token=self._token, repo_id="valid_org/test-model-tf-org" - ) + model.save_pretrained(tmp_dir, push_to_hub=True, token=self._token, repo_id="valid_org/test-model-tf-org") new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") models_equal = True diff --git a/tests/test_modeling_utils.py b/tests/test_modeling_utils.py index 1885fc671b..e457dc07a9 100755 --- a/tests/test_modeling_utils.py +++ b/tests/test_modeling_utils.py @@ -1162,7 +1162,7 @@ class ModelPushToHubTester(unittest.TestCase): vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 ) model = BertModel(config) - model.push_to_hub("test-model", use_auth_token=self._token) + model.push_to_hub("test-model", token=self._token) new_model = BertModel.from_pretrained(f"{USER}/test-model") for p1, p2 in zip(model.parameters(), new_model.parameters()): @@ -1173,7 +1173,7 @@ class ModelPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, repo_id="test-model", push_to_hub=True, use_auth_token=self._token) + model.save_pretrained(tmp_dir, repo_id="test-model", push_to_hub=True, token=self._token) new_model = BertModel.from_pretrained(f"{USER}/test-model") for p1, p2 in zip(model.parameters(), new_model.parameters()): @@ -1202,7 +1202,7 @@ The commit description supports markdown synthax see: vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 ) model = BertModel(config) - model.push_to_hub("valid_org/test-model-org", use_auth_token=self._token) + model.push_to_hub("valid_org/test-model-org", token=self._token) new_model = BertModel.from_pretrained("valid_org/test-model-org") for p1, p2 in zip(model.parameters(), new_model.parameters()): @@ -1213,9 +1213,7 @@ The commit description supports markdown synthax see: # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained( - tmp_dir, push_to_hub=True, use_auth_token=self._token, repo_id="valid_org/test-model-org" - ) + model.save_pretrained(tmp_dir, push_to_hub=True, token=self._token, repo_id="valid_org/test-model-org") new_model = BertModel.from_pretrained("valid_org/test-model-org") for p1, p2 in zip(model.parameters(), new_model.parameters()): @@ -1228,7 +1226,7 @@ The commit description supports markdown synthax see: config = CustomConfig(hidden_size=32) model = CustomModel(config) - model.push_to_hub("test-dynamic-model", use_auth_token=self._token) + model.push_to_hub("test-dynamic-model", token=self._token) # checks self.assertDictEqual( config.auto_map, diff --git a/tests/test_tokenization_utils.py b/tests/test_tokenization_utils.py index 2984de97fd..3f7f7249f9 100644 --- a/tests/test_tokenization_utils.py +++ b/tests/test_tokenization_utils.py @@ -146,7 +146,7 @@ class TokenizerPushToHubTester(unittest.TestCase): vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) tokenizer = BertTokenizer(vocab_file) - tokenizer.push_to_hub("test-tokenizer", use_auth_token=self._token) + tokenizer.push_to_hub("test-tokenizer", token=self._token) new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer") self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) @@ -155,7 +155,7 @@ class TokenizerPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: - tokenizer.save_pretrained(tmp_dir, repo_id="test-tokenizer", push_to_hub=True, use_auth_token=self._token) + tokenizer.save_pretrained(tmp_dir, repo_id="test-tokenizer", push_to_hub=True, token=self._token) new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer") self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) @@ -167,7 +167,7 @@ class TokenizerPushToHubTester(unittest.TestCase): vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) tokenizer = BertTokenizer(vocab_file) - tokenizer.push_to_hub("valid_org/test-tokenizer-org", use_auth_token=self._token) + tokenizer.push_to_hub("valid_org/test-tokenizer-org", token=self._token) new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org") self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) @@ -177,7 +177,7 @@ class TokenizerPushToHubTester(unittest.TestCase): # Push to hub via save_pretrained with tempfile.TemporaryDirectory() as tmp_dir: tokenizer.save_pretrained( - tmp_dir, repo_id="valid_org/test-tokenizer-org", push_to_hub=True, use_auth_token=self._token + tmp_dir, repo_id="valid_org/test-tokenizer-org", push_to_hub=True, token=self._token ) new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org") @@ -193,7 +193,7 @@ class TokenizerPushToHubTester(unittest.TestCase): tokenizer = CustomTokenizer(vocab_file) # No fast custom tokenizer - tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token) + tokenizer.push_to_hub("test-dynamic-tokenizer", token=self._token) tokenizer = AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer", trust_remote_code=True) # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module @@ -210,7 +210,7 @@ class TokenizerPushToHubTester(unittest.TestCase): bert_tokenizer.save_pretrained(tmp_dir) tokenizer = CustomTokenizerFast.from_pretrained(tmp_dir) - tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token) + tokenizer.push_to_hub("test-dynamic-tokenizer", token=self._token) tokenizer = AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer", trust_remote_code=True) # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module diff --git a/tests/utils/test_hub_utils.py b/tests/utils/test_hub_utils.py index 5ff1ed679e..dffc018e28 100644 --- a/tests/utils/test_hub_utils.py +++ b/tests/utils/test_hub_utils.py @@ -132,10 +132,10 @@ class GetFromCacheTests(unittest.TestCase): """Test download file from a gated repo fails with correct message when not authenticated.""" with self.assertRaisesRegex(EnvironmentError, "You are trying to access a gated repo."): # All files except README.md are protected on a gated repo. - cached_file(GATED_REPO, "gated_file.txt", use_auth_token=False) + cached_file(GATED_REPO, "gated_file.txt", token=False) def test_has_file_gated_repo(self): """Test check file existence from a gated repo fails with correct message when not authenticated.""" with self.assertRaisesRegex(EnvironmentError, "is a gated repository"): # All files except README.md are protected on a gated repo. - has_file(GATED_REPO, "gated_file.txt", use_auth_token=False) + has_file(GATED_REPO, "gated_file.txt", token=False)