Fix #34494 assistant tokens when truncated (#34531)

* Fix assistant tokens when truncated * fix test * fix test * step
2024-11-05 17:10:15 +02:00
parent 74d3824cc0
commit 082e57e0d4
8 changed files with 129 additions and 1 deletions
--- a/tests/models/markuplm/test_tokenization_markuplm.py
+++ b/tests/models/markuplm/test_tokenization_markuplm.py
@@ -2330,3 +2330,7 @@ class MarkupLMTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    @unittest.skip("Chat is not supported")
    def test_chat_template_return_assistant_tokens_mask(self):
        pass
+
+    @unittest.skip("Chat is not supported")
+    def test_chat_template_return_assistant_tokens_mask_truncated(self):
+        pass