Trainer - deprecate tokenizer for processing_class (#32385)

* Trainer - deprecate tokenizer for processing_class

* Extend chage across Seq2Seq trainer and docs

* Add tests

* Update to FutureWarning and add deprecation version
This commit is contained in:
amyeroberts
2024-10-02 14:08:46 +01:00
committed by GitHub
parent e7c8af7f33
commit b7474f211d
99 changed files with 569 additions and 442 deletions

View File

@@ -129,7 +129,7 @@ class Seq2seqTrainerTester(TestCasePlus):
compute_metrics=_compute_metrics,
train_dataset=train_dataset,
eval_dataset=val_dataset,
tokenizer=tokenizer,
processing_class=tokenizer,
)
# start training
@@ -158,7 +158,7 @@ class Seq2seqTrainerTester(TestCasePlus):
trainer = Seq2SeqTrainer(
model=model,
args=training_args,
tokenizer=tokenizer,
processing_class=tokenizer,
data_collator=data_collator,
compute_metrics=lambda x: {"samples": x[0].shape[0]},
)
@@ -199,7 +199,7 @@ class Seq2seqTrainerTester(TestCasePlus):
_ = Seq2SeqTrainer(
model=model,
args=training_args,
tokenizer=tokenizer,
processing_class=tokenizer,
data_collator=data_collator,
compute_metrics=lambda x: {"samples": x[0].shape[0]},
)