Refactor prepare_seq2seq_batch (#9524)
* Add target contextmanager and rework prepare_seq2seq_batch * Fix tests, treat BART and Barthez * Add last tokenizers * Fix test * Set src token before calling the superclass * Remove special behavior for T5 * Remove needless imports * Remove needless asserts
This commit is contained in:
@@ -508,12 +508,6 @@ class TestMarian_en_ROMANCE(MarianIntegrationTest):
|
||||
def test_batch_generation_en_ROMANCE_multi(self):
|
||||
self._assert_generated_batch_equal_expected()
|
||||
|
||||
def test_tokenizer_handles_empty(self):
|
||||
normalized = self.tokenizer.normalize("")
|
||||
self.assertIsInstance(normalized, str)
|
||||
with self.assertRaises(ValueError):
|
||||
self.tokenizer.prepare_seq2seq_batch([""], return_tensors="pt")
|
||||
|
||||
@slow
|
||||
def test_pipeline(self):
|
||||
device = 0 if torch_device == "cuda" else -1
|
||||
|
||||
Reference in New Issue
Block a user