SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix missing usage of token (#25382)

Browse Source 
* add missing tokens

* fix

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2023-08-08 16:27:24 +02:00
committed by GitHub
parent 5bd8c011bb
commit 9c7b744795
6 changed files with 96 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
examples/tensorflow/language-modeling/run_mlm.py
View File

@@ -358,12 +358,16 @@ def main():
# In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if checkpoint is not None:
config = AutoConfig.from_pretrained(checkpoint, trust_remote_code=model_args.trust_remote_code)
config = AutoConfig.from_pretrained(
checkpoint, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
elif model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name, trust_remote_code=model_args.trust_remote_code)
config = AutoConfig.from_pretrained(
model_args.config_name, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(
model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code
model_args.model_name_or_path, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
else:
config = CONFIG_MAPPING[model_args.model_type]()
@@ -371,11 +375,11 @@ def main():
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, trust_remote_code=model_args.trust_remote_code
model_args.tokenizer_name, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code
model_args.model_name_or_path, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
else:
raise ValueError(
@@ -512,15 +516,20 @@ def main():
# region Prepare model
if checkpoint is not None:
model = TFAutoModelForMaskedLM.from_pretrained(
checkpoint, config=config, trust_remote_code=model_args.trust_remote_code
checkpoint, config=config, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
elif model_args.model_name_or_path:
model = TFAutoModelForMaskedLM.from_pretrained(
model_args.model_name_or_path, config=config, trust_remote_code=model_args.trust_remote_code
model_args.model_name_or_path,
config=config,
token=model_args.token,
trust_remote_code=model_args.trust_remote_code,
)
else:
logger.info("Training new model from scratch")
model = TFAutoModelForMaskedLM.from_config(config, trust_remote_code=model_args.trust_remote_code)
model = TFAutoModelForMaskedLM.from_config(
config, token=model_args.token, trust_remote_code=model_args.trust_remote_code
)
# We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
# on a small vocab and want a smaller embedding size, remove this test.