From 458b0cd2c544cdd6c700f9b0c21077c889bcee6c Mon Sep 17 00:00:00 2001 From: Sai-Suraj-27 Date: Mon, 5 Aug 2024 19:49:42 +0530 Subject: [PATCH] fix: Updated `test_embeded_special_tokens` for luke and mluke models (#32413) Fixed tokenizertests for luke, mluke models. --- tests/models/luke/test_tokenization_luke.py | 4 +--- tests/models/mluke/test_tokenization_mluke.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/models/luke/test_tokenization_luke.py b/tests/models/luke/test_tokenization_luke.py index a7b544d460..a648f28d1f 100644 --- a/tests/models/luke/test_tokenization_luke.py +++ b/tests/models/luke/test_tokenization_luke.py @@ -146,11 +146,9 @@ class LukeTokenizerTest(TokenizerTesterMixin, unittest.TestCase): # token_type_ids should put 0 everywhere self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) - # token_type_ids should put 0 everywhere - self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) - # attention_mask should put 1 everywhere, so sum over length should be 1 self.assertEqual( + sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]), sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]), ) diff --git a/tests/models/mluke/test_tokenization_mluke.py b/tests/models/mluke/test_tokenization_mluke.py index bc9210c913..7af516849f 100644 --- a/tests/models/mluke/test_tokenization_mluke.py +++ b/tests/models/mluke/test_tokenization_mluke.py @@ -109,11 +109,9 @@ class MLukeTokenizerTest(TokenizerTesterMixin, unittest.TestCase): # token_type_ids should put 0 everywhere self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) - # token_type_ids should put 0 everywhere - self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) - # attention_mask should put 1 everywhere, so sum over length should be 1 self.assertEqual( + sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]), sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]), )