SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fsmt slow test uses lists (#8031)

Browse Source
This commit is contained in:
Sam Shleifer
2020-10-26 08:32:36 -04:00
committed by GitHub
parent 101186bc1f
commit f20aec1de5
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/test_tokenization_fsmt.py
View File

@@ -144,11 +144,11 @@ class FSMTTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
# for src_text, _ in targets: print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""") # for src_text, _ in targets: print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""")
for src_text, tgt_input_ids in targets: for src_text, tgt_input_ids in targets:
input_ids = tokenizer_enc.encode(src_text, return_tensors="pt")[0].tolist() encoded_ids = tokenizer_enc.encode(src_text, return_tensors=None)
self.assertListEqual(input_ids, tgt_input_ids) self.assertListEqual(encoded_ids, tgt_input_ids)
# and decode backward, using the reversed languages model # and decode backward, using the reversed languages model
decoded_text = tokenizer_dec.decode(input_ids, skip_special_tokens=True) decoded_text = tokenizer_dec.decode(encoded_ids, skip_special_tokens=True)
self.assertEqual(decoded_text, src_text) self.assertEqual(decoded_text, src_text)
@unittest.skip("FSMTConfig.__init__ requires non-optional args") @unittest.skip("FSMTConfig.__init__ requires non-optional args")