update examples after ruff being updated (#36972)
* update * update --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
@@ -438,7 +438,7 @@ def main():
|
||||
else:
|
||||
model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code)
|
||||
n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
|
||||
logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
|
||||
logger.info(f"Training new model from scratch - Total size={n_params / 2**20:.2f}M params")
|
||||
|
||||
# We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
|
||||
# on a small vocab and want a smaller embedding size, remove this test.
|
||||
|
||||
@@ -265,8 +265,7 @@ class DataTrainingArguments:
|
||||
default="<fim_pad>",
|
||||
metadata={
|
||||
"help": (
|
||||
"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True. "
|
||||
"Defaults to '<fim_pad>'."
|
||||
"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True. Defaults to '<fim_pad>'."
|
||||
)
|
||||
},
|
||||
)
|
||||
@@ -514,7 +513,7 @@ def main():
|
||||
attn_implementation=model_args.attn_implementation,
|
||||
)
|
||||
n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
|
||||
logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
|
||||
logger.info(f"Training new model from scratch - Total size={n_params / 2**20:.2f}M params")
|
||||
|
||||
# Add the new FIM tokens to the tokenizer and resize model's vocab embeddings
|
||||
special_tokens = [data_args.fim_prefix_token, data_args.fim_middle_token, data_args.fim_suffix_token]
|
||||
|
||||
@@ -234,9 +234,7 @@ def parse_args():
|
||||
"--fim_pad_token",
|
||||
type=str,
|
||||
default="<fim_pad>",
|
||||
help=(
|
||||
"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True." " Defaults to '<fim_pad>'."
|
||||
),
|
||||
help=("Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True. Defaults to '<fim_pad>'."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preprocessing_num_workers",
|
||||
|
||||
@@ -491,7 +491,7 @@ def main():
|
||||
# E.g. characters, such as `,` and `.` do not really have an acoustic characteristic
|
||||
# that could be easily picked up by the model
|
||||
chars_to_ignore_regex = (
|
||||
f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None
|
||||
f"[{''.join(data_args.chars_to_ignore)}]" if data_args.chars_to_ignore is not None else None
|
||||
)
|
||||
text_column_name = data_args.text_column_name
|
||||
|
||||
|
||||
@@ -471,7 +471,7 @@ def main():
|
||||
# E.g. characters, such as `,` and `.` do not really have an acoustic characteristic
|
||||
# that could be easily picked up by the model
|
||||
chars_to_ignore_regex = (
|
||||
f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None
|
||||
f"[{''.join(data_args.chars_to_ignore)}]" if data_args.chars_to_ignore is not None else None
|
||||
)
|
||||
text_column_name = data_args.text_column_name
|
||||
|
||||
|
||||
@@ -505,9 +505,9 @@ def main():
|
||||
return
|
||||
|
||||
if isinstance(tokenizer, tuple(MULTILINGUAL_TOKENIZERS)):
|
||||
assert (
|
||||
data_args.lang is not None
|
||||
), f"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument"
|
||||
assert data_args.lang is not None, (
|
||||
f"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument"
|
||||
)
|
||||
|
||||
tokenizer.src_lang = data_args.lang
|
||||
tokenizer.tgt_lang = data_args.lang
|
||||
|
||||
@@ -199,9 +199,9 @@ class DataTrainingArguments:
|
||||
train_extension = self.train_file.split(".")[-1]
|
||||
assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file."
|
||||
validation_extension = self.validation_file.split(".")[-1]
|
||||
assert (
|
||||
validation_extension == train_extension
|
||||
), "`validation_file` should have the same extension (csv or json) as `train_file`."
|
||||
assert validation_extension == train_extension, (
|
||||
"`validation_file` should have the same extension (csv or json) as `train_file`."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -357,9 +357,9 @@ def main():
|
||||
if data_args.test_file is not None:
|
||||
train_extension = data_args.train_file.split(".")[-1]
|
||||
test_extension = data_args.test_file.split(".")[-1]
|
||||
assert (
|
||||
test_extension == train_extension
|
||||
), "`test_file` should have the same extension (csv or json) as `train_file`."
|
||||
assert test_extension == train_extension, (
|
||||
"`test_file` should have the same extension (csv or json) as `train_file`."
|
||||
)
|
||||
data_files["test"] = data_args.test_file
|
||||
else:
|
||||
raise ValueError("Need either a dataset name or a test file for `do_predict`.")
|
||||
|
||||
@@ -156,9 +156,9 @@ class DataTrainingArguments:
|
||||
train_extension = self.train_file.split(".")[-1]
|
||||
assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file."
|
||||
validation_extension = self.validation_file.split(".")[-1]
|
||||
assert (
|
||||
validation_extension == train_extension
|
||||
), "`validation_file` should have the same extension (csv or json) as `train_file`."
|
||||
assert validation_extension == train_extension, (
|
||||
"`validation_file` should have the same extension (csv or json) as `train_file`."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -313,9 +313,9 @@ def main():
|
||||
if data_args.test_file is not None:
|
||||
train_extension = data_args.train_file.split(".")[-1]
|
||||
test_extension = data_args.test_file.split(".")[-1]
|
||||
assert (
|
||||
test_extension == train_extension
|
||||
), "`test_file` should have the same extension (csv or json) as `train_file`."
|
||||
assert test_extension == train_extension, (
|
||||
"`test_file` should have the same extension (csv or json) as `train_file`."
|
||||
)
|
||||
data_files["test"] = data_args.test_file
|
||||
else:
|
||||
raise ValueError("Need either a GLUE task or a test file for `do_predict`.")
|
||||
|
||||
@@ -322,7 +322,7 @@ def main():
|
||||
parser.add_argument(
|
||||
"--use_cpu",
|
||||
action="store_true",
|
||||
help="Whether or not to use cpu. If set to False, " "we will use gpu/npu or mps device if available",
|
||||
help="Whether or not to use cpu. If set to False, we will use gpu/npu or mps device if available",
|
||||
)
|
||||
parser.add_argument("--num_return_sequences", type=int, default=1, help="The number of samples to generate.")
|
||||
parser.add_argument(
|
||||
|
||||
@@ -68,7 +68,7 @@ def main():
|
||||
parser.add_argument(
|
||||
"--use_cpu",
|
||||
action="store_true",
|
||||
help="Whether or not to use cpu. If set to False, " "we will use gpu/npu or mps device if available",
|
||||
help="Whether or not to use cpu. If set to False, we will use gpu/npu or mps device if available",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fp16",
|
||||
|
||||
@@ -436,9 +436,9 @@ def main():
|
||||
|
||||
# Set decoder_start_token_id
|
||||
if model.config.decoder_start_token_id is None and isinstance(tokenizer, (MBartTokenizer, MBartTokenizerFast)):
|
||||
assert (
|
||||
args.target_lang is not None and args.source_lang is not None
|
||||
), "mBart requires --target_lang and --source_lang"
|
||||
assert args.target_lang is not None and args.source_lang is not None, (
|
||||
"mBart requires --target_lang and --source_lang"
|
||||
)
|
||||
if isinstance(tokenizer, MBartTokenizer):
|
||||
model.config.decoder_start_token_id = tokenizer.lang_code_to_id[args.target_lang]
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user