Fix #34494 assistant tokens when truncated (#34531)

* Fix assistant tokens when truncated * fix test * fix test * step
2024-11-05 17:10:15 +02:00
parent 74d3824cc0
commit 082e57e0d4
8 changed files with 129 additions and 1 deletions
--- a/tests/models/udop/test_tokenization_udop.py
+++ b/tests/models/udop/test_tokenization_udop.py
@@ -1161,6 +1161,10 @@ class UdopTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    def test_chat_template_return_assistant_tokens_mask(self):
        pass

+    @unittest.skip("Chat is not supported")
+    def test_chat_template_return_assistant_tokens_mask_truncated(self):
+        pass
+
    @unittest.skip(reason="Chat template tests don't play well with table/layout models.")
    def test_chat_template_batched(self):
        pass