From 4e441e529c297dcfed0c780f56e6b9da4e9e8281 Mon Sep 17 00:00:00 2001 From: Thibault Douzon Date: Mon, 3 Apr 2023 16:32:36 +0200 Subject: [PATCH] =?UTF-8?q?fix=20LayoutLMv3TokenizerFast=20subword=20label?= =?UTF-8?q?=20after=20'=C4=A0'=20token=20(#21695)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LayoutLMv3TokenizerFast produces empty 'Ġ' token with `offset_mapping = (0, 0)`. Next token is wrongly assumed to also be beginning of word and isn't correctly assigned `pad_token_label`. Modify test with text that produce 'Ġ' token. Remove copy check from LayoutLMv2TokenizerFast for `_batch_encode_plus`. solves issue: #19978 --- .../layoutlmv3/tokenization_layoutlmv3_fast.py | 8 ++++++-- .../layoutlmv3/test_tokenization_layoutlmv3.py | 12 ++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py b/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py index 4bd3e91480..3d7445e449 100644 --- a/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py +++ b/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py @@ -508,7 +508,6 @@ class LayoutLMv3TokenizerFast(PreTrainedTokenizerFast): **kwargs, ) - # Copied from transformers.models.layoutlmv2.tokenization_layoutlmv2_fast.LayoutLMv2TokenizerFast._batch_encode_plus with LayoutLMv2->LayoutLMv3 def _batch_encode_plus( self, batch_text_or_text_pairs: Union[ @@ -640,6 +639,7 @@ class LayoutLMv3TokenizerFast(PreTrainedTokenizerFast): else: original_index = batch_index labels_example = [] + previous_token_empty = False for id, offset, word_id in zip( sanitized_tokens["input_ids"][batch_index], sanitized_tokens["offset_mapping"][batch_index], @@ -647,11 +647,15 @@ class LayoutLMv3TokenizerFast(PreTrainedTokenizerFast): ): if word_id is not None: if self.only_label_first_subword: - if offset[0] == 0: + if offset[0] == 0 and not previous_token_empty: # Use the real label id for the first token of the word, and padding ids for the remaining tokens labels_example.append(word_labels[original_index][word_id]) else: labels_example.append(self.pad_token_label) + if offset == (0, 0): + previous_token_empty = True + else: + previous_token_empty = False else: labels_example.append(word_labels[original_index][word_id]) else: diff --git a/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py b/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py index 884f876803..63d86f280c 100644 --- a/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py +++ b/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py @@ -2277,14 +2277,14 @@ class LayoutLMv3TokenizationTest(TokenizerTesterMixin, unittest.TestCase): @slow def test_only_label_first_subword(self): - words = ["hello", "niels"] + words = ["hello", "niels", "0000000000000000"] boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))] - word_labels = [0, 1] + word_labels = [0, 1, 2] # test slow tokenizer tokenizer_p = LayoutLMv3Tokenizer.from_pretrained("microsoft/layoutlmv3-base", add_visual_labels=False) encoding = tokenizer_p(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, -100, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, -100, 2, -100, -100]) tokenizer_p = LayoutLMv3Tokenizer.from_pretrained( "microsoft/layoutlmv3-base", @@ -2292,12 +2292,12 @@ class LayoutLMv3TokenizationTest(TokenizerTesterMixin, unittest.TestCase): add_visual_labels=False, ) encoding = tokenizer_p(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, 1, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, 1, 2, 2, -100]) # test fast tokenizer tokenizer_r = LayoutLMv3TokenizerFast.from_pretrained("microsoft/layoutlmv3-base", add_visual_labels=False) encoding = tokenizer_r(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, -100, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, -100, 2, -100, -100]) tokenizer_r = LayoutLMv3Tokenizer.from_pretrained( "microsoft/layoutlmv3-base", @@ -2305,7 +2305,7 @@ class LayoutLMv3TokenizationTest(TokenizerTesterMixin, unittest.TestCase): add_visual_labels=False, ) encoding = tokenizer_r(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, 1, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, 1, 2, 2, -100]) @slow def test_layoutlmv3_integration_test(self):