SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove deprecated code (#37059)

Browse Source 
* Remove deprecated code

* fix get_loading_attributes

* fix error

* skip test

---------

Co-authored-by: Marc Sun <57196510+SunMarc@users.noreply.github.com>
Co-authored-by: Mohamed Mekkouri <93391238+MekkCyber@users.noreply.github.com>
This commit is contained in:
cyyever
2025-03-31 17:15:35 +08:00
committed by GitHub
parent d1efaf0318
commit f99c279d20
8 changed files with 13 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/pytorch/language-modeling/run_fim.py
View File

@@ -47,7 +47,7 @@ from transformers import (
Trainer,
TrainingArguments,
default_data_collator,
is_torch_tpu_available,
is_torch_xla_available,
set_seed,
)
from transformers.integrations import is_deepspeed_zero3_enabled
@@ -525,7 +525,7 @@ def main():
if torch.cuda.is_availble():
pad_factor = 8
elif is_torch_tpu_available():
elif is_torch_xla_available(check_is_tpu=True):
pad_factor = 128
# Add the new tokens to the tokenizer
@@ -795,9 +795,13 @@ def main():
processing_class=tokenizer,
# Data collator will default to DataCollatorWithPadding, so we change it.
data_collator=default_data_collator,
compute_metrics=compute_metrics if training_args.do_eval and not is_torch_tpu_available() else None,
compute_metrics=compute_metrics
if training_args.do_eval and not is_torch_xla_available(check_is_tpu=True)
else None,
preprocess_logits_for_metrics=(
preprocess_logits_for_metrics if training_args.do_eval and not is_torch_tpu_available() else None
preprocess_logits_for_metrics
if training_args.do_eval and not is_torch_xla_available(check_is_tpu=True)
else None
),
)

4
examples/pytorch/language-modeling/run_fim_no_trainer.py
View File

@@ -52,7 +52,7 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
is_torch_tpu_available,
is_torch_xla_available,
)
from transformers.integrations import is_deepspeed_zero3_enabled
from transformers.utils import check_min_version, send_example_telemetry
@@ -492,7 +492,7 @@ def main():
if torch.cuda.is_availble():
pad_factor = 8
elif is_torch_tpu_available():
elif is_torch_xla_available(check_is_tpu=True):
pad_factor = 128
# Add the new tokens to the tokenizer