SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

🚨🚨🚨Deprecate evaluation_strategy to eval_strategy🚨🚨🚨 (#30190)

Browse Source 
* Alias

* Note alias

* Tests and src

* Rest

* Clean

* Change typing?

* Fix tests

* Deprecation versions
This commit is contained in:
Zach Mueller
2024-04-18 12:49:43 -04:00
committed by GitHub
parent c86d020ead
commit 60d5f8f9f0
116 changed files with 214 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/deepspeed/test_deepspeed.py
View File

@@ -959,7 +959,7 @@ class TrainerIntegrationDeepSpeed(TrainerIntegrationDeepSpeedWithCustomConfig, T
"do_train": True,
"do_eval": True,
"optim": "adafactor",
"evaluation_strategy": "steps",
"eval_strategy": "steps",
"eval_steps": 1,
"save_strategy": "steps",
"save_steps": 1,

2
tests/extended/test_trainer_ext.py
View File

@@ -308,7 +308,7 @@ class TestTrainerExt(TestCasePlus):
--per_device_eval_batch_size 4
--max_eval_samples 8
--val_max_target_length {max_len}
--evaluation_strategy steps
--eval_strategy steps
--eval_steps {str(eval_steps)}
""".split()

2
tests/fsdp/test_fsdp.py
View File

@@ -308,6 +308,6 @@ class TrainerIntegrationFSDP(TestCasePlus, TrainerIntegrationCommon):
--logging_steps {logging_steps}
--save_strategy epoch
--do_eval
--evaluation_strategy epoch
--eval_strategy epoch
--report_to none
"""

32
tests/trainer/test_trainer.py
View File

@@ -740,7 +740,7 @@ class TrainerIntegrationPrerunTest(TestCasePlus, TrainerIntegrationCommon):
eval_dataset = RegressionDataset(length=64)
args = TrainingArguments(
"./regression",
evaluation_strategy="epoch",
eval_strategy="epoch",
metric_for_best_model="eval_loss",
)
model = RegressionModel()
@@ -772,7 +772,7 @@ class TrainerIntegrationPrerunTest(TestCasePlus, TrainerIntegrationCommon):
args = TrainingArguments(
"./regression",
lr_scheduler_type="reduce_lr_on_plateau",
evaluation_strategy="epoch",
eval_strategy="epoch",
metric_for_best_model="eval_loss",
num_train_epochs=10,
learning_rate=0.2,
@@ -2210,7 +2210,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
output_dir=tmpdir,
learning_rate=0.1,
eval_steps=5,
evaluation_strategy="steps",
eval_strategy="steps",
save_steps=5,
load_best_model_at_end=True,
)
@@ -2226,7 +2226,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
output_dir=tmpdir,
learning_rate=0.1,
eval_steps=5,
evaluation_strategy="steps",
eval_strategy="steps",
save_steps=5,
load_best_model_at_end=True,
metric_for_best_model="accuracy",
@@ -2243,7 +2243,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
b=2.5,
output_dir=tmpdir,
learning_rate=0.1,
evaluation_strategy="epoch",
eval_strategy="epoch",
save_strategy="epoch",
load_best_model_at_end=True,
metric_for_best_model="accuracy",
@@ -2262,7 +2262,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
output_dir=tmpdir,
learning_rate=0.1,
eval_steps=5,
evaluation_strategy="steps",
eval_strategy="steps",
save_steps=5,
load_best_model_at_end=True,
pretrained=False,
@@ -2283,7 +2283,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
output_dir=tmpdir,
learning_rate=0.1,
eval_steps=5,
evaluation_strategy="steps",
eval_strategy="steps",
save_steps=5,
load_best_model_at_end=True,
save_safetensors=save_safetensors,
@@ -2437,7 +2437,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
gradient_accumulation_steps=1,
per_device_train_batch_size=16,
load_best_model_at_end=True,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
save_strategy=IntervalStrategy.EPOCH,
compute_metrics=AlmostAccuracy(),
metric_for_best_model="accuracy",
@@ -2453,7 +2453,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
num_train_epochs=20,
gradient_accumulation_steps=1,
per_device_train_batch_size=16,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
compute_metrics=AlmostAccuracy(),
metric_for_best_model="accuracy",
)
@@ -2497,7 +2497,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
# With best model at end
trainer = get_regression_trainer(
output_dir=tmp_dir, evaluation_strategy="steps", load_best_model_at_end=True, save_total_limit=2
output_dir=tmp_dir, eval_strategy="steps", load_best_model_at_end=True, save_total_limit=2
)
trainer.state.best_model_checkpoint = os.path.join(tmp_dir, "checkpoint-5")
self.check_checkpoint_deletion(trainer, tmp_dir, [5, 25])
@@ -2505,7 +2505,7 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
# Edge case: we don't always honor save_total_limit=1 if load_best_model_at_end=True to be able to resume
# from checkpoint
trainer = get_regression_trainer(
output_dir=tmp_dir, evaluation_strategy="steps", load_best_model_at_end=True, save_total_limit=1
output_dir=tmp_dir, eval_strategy="steps", load_best_model_at_end=True, save_total_limit=1
)
trainer.state.best_model_checkpoint = os.path.join(tmp_dir, "checkpoint-25")
self.check_checkpoint_deletion(trainer, tmp_dir, [25])
@@ -3341,7 +3341,7 @@ class TrainerHyperParameterOptunaIntegrationTest(unittest.TestCase):
output_dir=tmp_dir,
learning_rate=0.1,
logging_steps=1,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
save_strategy=IntervalStrategy.EPOCH,
num_train_epochs=4,
disable_tqdm=True,
@@ -3390,7 +3390,7 @@ class TrainerHyperParameterMultiObjectOptunaIntegrationTest(unittest.TestCase):
output_dir=tmp_dir,
learning_rate=0.1,
logging_steps=1,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
save_strategy=IntervalStrategy.EPOCH,
num_train_epochs=10,
disable_tqdm=True,
@@ -3448,7 +3448,7 @@ class TrainerHyperParameterRayIntegrationTest(unittest.TestCase):
output_dir=tmp_dir,
learning_rate=0.1,
logging_steps=1,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
save_strategy=IntervalStrategy.EPOCH,
num_train_epochs=4,
disable_tqdm=True,
@@ -3511,7 +3511,7 @@ class TrainerHyperParameterSigOptIntegrationTest(unittest.TestCase):
output_dir=tmp_dir,
learning_rate=0.1,
logging_steps=1,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
save_strategy=IntervalStrategy.EPOCH,
num_train_epochs=4,
disable_tqdm=True,
@@ -3931,7 +3931,7 @@ class TrainerHyperParameterWandbIntegrationTest(unittest.TestCase):
output_dir=tmp_dir,
learning_rate=0.1,
logging_steps=1,
evaluation_strategy=IntervalStrategy.EPOCH,
eval_strategy=IntervalStrategy.EPOCH,
save_strategy=IntervalStrategy.EPOCH,
num_train_epochs=4,
disable_tqdm=True,

10
tests/trainer/test_trainer_callback.py
View File

@@ -133,12 +133,12 @@ class TrainerCallbackTest(unittest.TestCase):
expected_events += ["on_step_begin", "on_step_end"]
if step % trainer.args.logging_steps == 0:
expected_events.append("on_log")
if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
if trainer.args.eval_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
expected_events += evaluation_events.copy()
if step % trainer.args.save_steps == 0:
expected_events.append("on_save")
expected_events.append("on_epoch_end")
if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
if trainer.args.eval_strategy == IntervalStrategy.EPOCH:
expected_events += evaluation_events.copy()
expected_events += ["on_log", "on_train_end"]
return expected_events
@@ -215,12 +215,12 @@ class TrainerCallbackTest(unittest.TestCase):
events = trainer.callback_handler.callbacks[-2].events
self.assertEqual(events, self.get_expected_events(trainer))
trainer = self.get_trainer(callbacks=[MyTestTrainerCallback], eval_steps=5, evaluation_strategy="steps")
trainer = self.get_trainer(callbacks=[MyTestTrainerCallback], eval_steps=5, eval_strategy="steps")
trainer.train()
events = trainer.callback_handler.callbacks[-2].events
self.assertEqual(events, self.get_expected_events(trainer))
trainer = self.get_trainer(callbacks=[MyTestTrainerCallback], evaluation_strategy="epoch")
trainer = self.get_trainer(callbacks=[MyTestTrainerCallback], eval_strategy="epoch")
trainer.train()
events = trainer.callback_handler.callbacks[-2].events
self.assertEqual(events, self.get_expected_events(trainer))
@@ -231,7 +231,7 @@ class TrainerCallbackTest(unittest.TestCase):
logging_steps=3,
save_steps=10,
eval_steps=5,
evaluation_strategy="steps",
eval_strategy="steps",
)
trainer.train()
events = trainer.callback_handler.callbacks[-2].events

2
tests/trainer/test_trainer_seq2seq.py
View File

@@ -113,7 +113,7 @@ class Seq2seqTrainerTester(TestCasePlus):
per_device_train_batch_size=batch_size,
per_device_eval_batch_size=batch_size,
predict_with_generate=True,
evaluation_strategy="steps",
eval_strategy="steps",
do_train=True,
do_eval=True,
warmup_steps=0,