SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove-auth-token (#27060)

Browse Source 
* don't use `use_auth_token`internally

* let's use token everywhere

* fixup
This commit is contained in:
Arthur
2023-11-13 14:20:54 +01:00
committed by GitHub
parent 8f577dca4f
commit b97cab7e6d
29 changed files with 93 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/research_projects/robust-speech-event/eval.py
View File

@@ -65,7 +65,7 @@ def normalize_text(text: str) -> str:
def main(args):
# load dataset
dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
dataset = load_dataset(args.dataset, args.config, split=args.split, token=True)
# for testing: only process the first two examples as a test
# dataset = dataset.select(range(10))

12
examples/research_projects/robust-speech-event/run_speech_recognition_ctc_bnb.py
View File

@@ -418,7 +418,7 @@ def main():
data_args.dataset_name,
data_args.dataset_config_name,
split=data_args.train_split_name,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
)
if data_args.audio_column_name not in raw_datasets["train"].column_names:
@@ -443,7 +443,7 @@ def main():
data_args.dataset_name,
data_args.dataset_config_name,
split=data_args.eval_split_name,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
)
if data_args.max_eval_samples is not None:
@@ -481,7 +481,7 @@ def main():
# the tokenizer
# load config
config = AutoConfig.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token
)
# 4. Next, if no tokenizer file is defined,
@@ -532,11 +532,11 @@ def main():
# load feature_extractor and tokenizer
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name_or_path,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
**tokenizer_kwargs,
)
feature_extractor = AutoFeatureExtractor.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token
)
# adapt config
@@ -564,7 +564,7 @@ def main():
model_args.model_name_or_path,
cache_dir=model_args.cache_dir,
config=config,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
)
# freeze encoder

12
examples/research_projects/robust-speech-event/run_speech_recognition_ctc_streaming.py
View File

@@ -395,7 +395,7 @@ def main():
# so we just need to set the correct target sampling rate and normalize the input
# via the `feature_extractor`
feature_extractor = AutoFeatureExtractor.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token
)
if training_args.do_train:
@@ -403,7 +403,7 @@ def main():
path=data_args.dataset_name,
name=data_args.dataset_config_name,
split=data_args.train_split_name,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
streaming=True,
sampling_rate=feature_extractor.sampling_rate,
)
@@ -431,7 +431,7 @@ def main():
path=data_args.dataset_name,
name=data_args.dataset_config_name,
split=data_args.eval_split_name,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
streaming=True,
sampling_rate=feature_extractor.sampling_rate,
)
@@ -465,7 +465,7 @@ def main():
# 3. Next, let's load the config as we might need it to create
# the tokenizer
config = AutoConfig.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
model_args.model_name_or_path, cache_dir=model_args.cache_dir, token=data_args.use_auth_token
)
# 4. Now we can instantiate the tokenizer and model
@@ -481,7 +481,7 @@ def main():
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name_or_path,
config=config,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
)
# adapt config
@@ -509,7 +509,7 @@ def main():
model_args.model_name_or_path,
cache_dir=model_args.cache_dir,
config=config,
use_auth_token=data_args.use_auth_token,
token=data_args.use_auth_token,
)
# freeze encoder