Fix conflicting key in init kwargs in PreTrainedTokenizerBase (#31233)

* Fix conflicting key in init kwargs in PreTrainedTokenizerBase

* Update code to check for callable key in save_pretrained

* Apply PR suggestions

* Invoke CI

* Updates based on PR suggestion
This commit is contained in:
Omar Salman
2024-08-01 17:32:13 +05:00
committed by GitHub
parent db8c7caeb6
commit b4727a1216
2 changed files with 12 additions and 0 deletions

View File

@@ -4408,3 +4408,11 @@ class TokenizerTesterMixin:
replace_additional_special_tokens=False,
)
self.assertEqual(tokenizer_2.additional_special_tokens, ["<other>", "<another>", "<tok>"])
def test_tokenizer_initialization_with_conflicting_key(self):
get_tokenizer_func = self.get_rust_tokenizer if self.test_rust_tokenizer else self.get_tokenizer
with self.assertRaises(AttributeError, msg="conflicts with the method"):
get_tokenizer_func(add_special_tokens=True)
with self.assertRaises(AttributeError, msg="conflicts with the method"):
get_tokenizer_func(get_vocab=True)