SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Allow trust_remote_code in example scripts (#25248)

Browse Source 
* pytorch examples

* pytorch mim no trainer

* cookiecutter

* flax examples

* missed line in pytorch run_glue

* tensorflow examples

* tensorflow run_clip

* tensorflow run_mlm

* tensorflow run_ner

* tensorflow run_clm

* pytorch example from_configs

* pytorch no trainer examples

* Revert "tensorflow run_clip"

This reverts commit 261f86ac1f1c9e05dd3fd0291e1a1f8e573781d5.

* fix: duplicated argument
This commit is contained in:
Jackmin801
2023-08-07 22:32:25 +08:00
committed by GitHub
parent 65001cb1c8
commit 145109382a
49 changed files with 790 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
examples/tensorflow/token-classification/run_ner.py
View File

@@ -91,6 +91,16 @@ class ModelArguments:
"help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token`."
},
)
trust_remote_code: bool = field(
default=False,
metadata={
"help": (
"Whether or not to allow for custom models defined on the Hub in their own modeling files. This option"
"should only be set to `True` for repositories you trust and in which you have read the code, as it will"
"execute code present on the Hub on your local machine."
)
},
)
@dataclass
@@ -304,9 +314,17 @@ def main():
# In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name, num_labels=num_labels)
config = AutoConfig.from_pretrained(
model_args.config_name,
num_labels=num_labels,
trust_remote_code=model_args.trust_remote_code,
)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path, num_labels=num_labels)
config = AutoConfig.from_pretrained(
model_args.model_name_or_path,
num_labels=num_labels,
trust_remote_code=model_args.trust_remote_code,
)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning("You are instantiating a new config instance from scratch.")
@@ -319,9 +337,18 @@ def main():
)
if config.model_type in {"gpt2", "roberta"}:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True, add_prefix_space=True)
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name_or_path,
use_fast=True,
add_prefix_space=True,
trust_remote_code=model_args.trust_remote_code,
)
else:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True)
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name_or_path,
use_fast=True,
trust_remote_code=model_args.trust_remote_code,
)
# endregion
# region Preprocessing the raw datasets
@@ -392,10 +419,13 @@ def main():
model = TFAutoModelForTokenClassification.from_pretrained(
model_args.model_name_or_path,
config=config,
trust_remote_code=model_args.trust_remote_code,
)
else:
logger.info("Training new model from scratch")
model = TFAutoModelForTokenClassification.from_config(config)
model = TFAutoModelForTokenClassification.from_config(
config, trust_remote_code=model_args.trust_remote_code
)
# We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
# on a small vocab and want a smaller embedding size, remove this test.