From 7915a25976a6000f71b9197d963e56868f42e77e Mon Sep 17 00:00:00 2001
From: Pavel Iakubovskii <qubvel@gmail.com>
Date: Thu, 18 Apr 2024 09:30:40 +0100
Subject: [PATCH] Fix donut token2json multiline (#30300)

* Fix multiline processing

* Update test for token2json
---
 src/transformers/models/donut/processing_donut.py | 4 +++-
 tests/models/donut/test_processing_donut.py       | 4 ++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/transformers/models/donut/processing_donut.py b/src/transformers/models/donut/processing_donut.py
index 5636ecb943..1f03fd6306 100644
--- a/src/transformers/models/donut/processing_donut.py
+++ b/src/transformers/models/donut/processing_donut.py
@@ -149,7 +149,9 @@ class DonutProcessor(ProcessorMixin):
                 end_token = end_token.group()
                 start_token_escaped = re.escape(start_token)
                 end_token_escaped = re.escape(end_token)
-                content = re.search(f"{start_token_escaped}(.*?){end_token_escaped}", tokens, re.IGNORECASE)
+                content = re.search(
+                    f"{start_token_escaped}(.*?){end_token_escaped}", tokens, re.IGNORECASE | re.DOTALL
+                )
                 if content is not None:
                     content = content.group(1).strip()
                     if r"<s_" in content and r"</s_" in content:  # non-leaf node
diff --git a/tests/models/donut/test_processing_donut.py b/tests/models/donut/test_processing_donut.py
index cad0e37bc5..c4ee25b252 100644
--- a/tests/models/donut/test_processing_donut.py
+++ b/tests/models/donut/test_processing_donut.py
@@ -35,6 +35,8 @@ class DonutProcessorTest(unittest.TestCase):
             "zip": "30301",
             "phone": "123-4567",
             "nicknames": [{"nickname": "Johnny"}, {"nickname": "JD"}],
+            "multiline": "text\nwith\nnewlines",
+            "empty": "",
         }
 
         sequence = (
@@ -42,6 +44,8 @@ class DonutProcessorTest(unittest.TestCase):
             "<s_state>GA</s_state><s_zip>30301</s_zip><s_phone>123-4567</s_phone>"
             "<s_nicknames><s_nickname>Johnny</s_nickname>"
             "<sep/><s_nickname>JD</s_nickname></s_nicknames>"
+            "<s_multiline>text\nwith\nnewlines</s_multiline>"
+            "<s_empty></s_empty>"
         )
         actual_json = self.processor.token2json(sequence)