SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Faster pegasus tokenization test with reduced data size (#7762)

Browse Source
This commit is contained in:
Sam Shleifer
2020-10-13 16:22:29 -04:00
committed by GitHub
parent 2d6e2ad4fa
commit 2977bd528f
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_tokenization_pegasus.py
View File

@@ -57,7 +57,7 @@ class PegasusTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
@require_torch @require_torch
def test_pegasus_large_seq2seq_truncation(self): def test_pegasus_large_seq2seq_truncation(self):
src_texts = ["This is going to be way too long" * 10000, "short example"] src_texts = ["This is going to be way too long." * 150, "short example"]
tgt_texts = ["not super long but more than 5 tokens", "tiny"] tgt_texts = ["not super long but more than 5 tokens", "tiny"]
batch = self.pegasus_large_tokenizer.prepare_seq2seq_batch(src_texts, tgt_texts=tgt_texts, max_target_length=5) batch = self.pegasus_large_tokenizer.prepare_seq2seq_batch(src_texts, tgt_texts=tgt_texts, max_target_length=5)
assert batch.input_ids.shape == (2, 1024) assert batch.input_ids.shape == (2, 1024)