SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix: Updated test_embeded_special_tokens for luke and mluke models (#32413)

Browse Source 
Fixed tokenizertests for luke, mluke models.
This commit is contained in:
Sai-Suraj-27
2024-08-05 19:49:42 +05:30
committed by GitHub
parent baf7e5c927
commit 458b0cd2c5
2 changed files with 2 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/models/luke/test_tokenization_luke.py
View File

@@ -146,11 +146,9 @@ class LukeTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
# token_type_ids should put 0 everywhere # token_type_ids should put 0 everywhere
self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
# token_type_ids should put 0 everywhere
self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
# attention_mask should put 1 everywhere, so sum over length should be 1 # attention_mask should put 1 everywhere, so sum over length should be 1
self.assertEqual( self.assertEqual(
sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]),
sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]), sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]),
) )

4
tests/models/mluke/test_tokenization_mluke.py
View File

@@ -109,11 +109,9 @@ class MLukeTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
# token_type_ids should put 0 everywhere # token_type_ids should put 0 everywhere
self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
# token_type_ids should put 0 everywhere
self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
# attention_mask should put 1 everywhere, so sum over length should be 1 # attention_mask should put 1 everywhere, so sum over length should be 1
self.assertEqual( self.assertEqual(
sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]),
sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]), sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]),
) )