update examples after ruff being updated (#36972)

* update

* update

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2025-03-25 18:15:47 +01:00
committed by GitHub
parent a41677a68b
commit 121830ab47
20 changed files with 42 additions and 45 deletions

View File

@@ -438,7 +438,7 @@ def main():
else:
model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code)
n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
logger.info(f"Training new model from scratch - Total size={n_params / 2**20:.2f}M params")
# We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
# on a small vocab and want a smaller embedding size, remove this test.

View File

@@ -265,8 +265,7 @@ class DataTrainingArguments:
default="<fim_pad>",
metadata={
"help": (
"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True. "
"Defaults to '<fim_pad>'."
"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True. Defaults to '<fim_pad>'."
)
},
)
@@ -514,7 +513,7 @@ def main():
attn_implementation=model_args.attn_implementation,
)
n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
logger.info(f"Training new model from scratch - Total size={n_params / 2**20:.2f}M params")
# Add the new FIM tokens to the tokenizer and resize model's vocab embeddings
special_tokens = [data_args.fim_prefix_token, data_args.fim_middle_token, data_args.fim_suffix_token]

View File

@@ -234,9 +234,7 @@ def parse_args():
"--fim_pad_token",
type=str,
default="<fim_pad>",
help=(
"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True." " Defaults to '<fim_pad>'."
),
help=("Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True. Defaults to '<fim_pad>'."),
)
parser.add_argument(
"--preprocessing_num_workers",

View File

@@ -491,7 +491,7 @@ def main():
# E.g. characters, such as `,` and `.` do not really have an acoustic characteristic
# that could be easily picked up by the model
chars_to_ignore_regex = (
f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None
f"[{''.join(data_args.chars_to_ignore)}]" if data_args.chars_to_ignore is not None else None
)
text_column_name = data_args.text_column_name

View File

@@ -471,7 +471,7 @@ def main():
# E.g. characters, such as `,` and `.` do not really have an acoustic characteristic
# that could be easily picked up by the model
chars_to_ignore_regex = (
f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None
f"[{''.join(data_args.chars_to_ignore)}]" if data_args.chars_to_ignore is not None else None
)
text_column_name = data_args.text_column_name

View File

@@ -505,9 +505,9 @@ def main():
return
if isinstance(tokenizer, tuple(MULTILINGUAL_TOKENIZERS)):
assert (
data_args.lang is not None
), f"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument"
assert data_args.lang is not None, (
f"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument"
)
tokenizer.src_lang = data_args.lang
tokenizer.tgt_lang = data_args.lang

View File

@@ -199,9 +199,9 @@ class DataTrainingArguments:
train_extension = self.train_file.split(".")[-1]
assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file."
validation_extension = self.validation_file.split(".")[-1]
assert (
validation_extension == train_extension
), "`validation_file` should have the same extension (csv or json) as `train_file`."
assert validation_extension == train_extension, (
"`validation_file` should have the same extension (csv or json) as `train_file`."
)
@dataclass
@@ -357,9 +357,9 @@ def main():
if data_args.test_file is not None:
train_extension = data_args.train_file.split(".")[-1]
test_extension = data_args.test_file.split(".")[-1]
assert (
test_extension == train_extension
), "`test_file` should have the same extension (csv or json) as `train_file`."
assert test_extension == train_extension, (
"`test_file` should have the same extension (csv or json) as `train_file`."
)
data_files["test"] = data_args.test_file
else:
raise ValueError("Need either a dataset name or a test file for `do_predict`.")

View File

@@ -156,9 +156,9 @@ class DataTrainingArguments:
train_extension = self.train_file.split(".")[-1]
assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file."
validation_extension = self.validation_file.split(".")[-1]
assert (
validation_extension == train_extension
), "`validation_file` should have the same extension (csv or json) as `train_file`."
assert validation_extension == train_extension, (
"`validation_file` should have the same extension (csv or json) as `train_file`."
)
@dataclass
@@ -313,9 +313,9 @@ def main():
if data_args.test_file is not None:
train_extension = data_args.train_file.split(".")[-1]
test_extension = data_args.test_file.split(".")[-1]
assert (
test_extension == train_extension
), "`test_file` should have the same extension (csv or json) as `train_file`."
assert test_extension == train_extension, (
"`test_file` should have the same extension (csv or json) as `train_file`."
)
data_files["test"] = data_args.test_file
else:
raise ValueError("Need either a GLUE task or a test file for `do_predict`.")

View File

@@ -322,7 +322,7 @@ def main():
parser.add_argument(
"--use_cpu",
action="store_true",
help="Whether or not to use cpu. If set to False, " "we will use gpu/npu or mps device if available",
help="Whether or not to use cpu. If set to False, we will use gpu/npu or mps device if available",
)
parser.add_argument("--num_return_sequences", type=int, default=1, help="The number of samples to generate.")
parser.add_argument(

View File

@@ -68,7 +68,7 @@ def main():
parser.add_argument(
"--use_cpu",
action="store_true",
help="Whether or not to use cpu. If set to False, " "we will use gpu/npu or mps device if available",
help="Whether or not to use cpu. If set to False, we will use gpu/npu or mps device if available",
)
parser.add_argument(
"--fp16",

View File

@@ -436,9 +436,9 @@ def main():
# Set decoder_start_token_id
if model.config.decoder_start_token_id is None and isinstance(tokenizer, (MBartTokenizer, MBartTokenizerFast)):
assert (
args.target_lang is not None and args.source_lang is not None
), "mBart requires --target_lang and --source_lang"
assert args.target_lang is not None and args.source_lang is not None, (
"mBart requires --target_lang and --source_lang"
)
if isinstance(tokenizer, MBartTokenizer):
model.config.decoder_start_token_id = tokenizer.lang_code_to_id[args.target_lang]
else: