Return assistant generated tokens mask in apply_chat_template (#30650)

return assistant generated tokens mask in apply_chat_template
2024-07-22 20:24:43 +03:00
parent 7987710696
commit 74d0eb3fed
8 changed files with 265 additions and 8 deletions
--- a/tests/models/markuplm/test_tokenization_markuplm.py
+++ b/tests/models/markuplm/test_tokenization_markuplm.py
@@ -2316,3 +2316,7 @@ class MarkupLMTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    @unittest.skip(reason="The model tested fails `Hub -> Fast == Hub -> Slow`, nothing much we can do")
    def test_added_tokens_serialization(self):
        pass
+
+    @unittest.skip("Chat is not supported")
+    def test_chat_template_return_assistant_tokens_mask(self):
+        pass