From 870e6f29a64b826249d3f3b8fc835cd4bf00f862 Mon Sep 17 00:00:00 2001 From: Patrick Deutschmann Date: Wed, 4 May 2022 18:23:37 +0200 Subject: [PATCH] Fix DeBERTa `token_type_ids` (#17082) --- src/transformers/convert_slow_tokenizer.py | 2 +- src/transformers/models/deberta/tokenization_deberta.py | 2 +- .../models/deberta/tokenization_deberta_fast.py | 4 ++-- tests/models/deberta/test_tokenization_deberta.py | 6 ++++++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 4f41f6b61f..6f32a4456e 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -407,7 +407,7 @@ class DebertaConverter(Converter): tokenizer.decoder = decoders.ByteLevel() tokenizer.post_processor = processors.TemplateProcessing( single="[CLS]:0 $A:0 [SEP]:0", - pair="[CLS]:0 $A:0 [SEP]:0 $B:0 [SEP]:0", + pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1", special_tokens=[ ("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")), ("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")), diff --git a/src/transformers/models/deberta/tokenization_deberta.py b/src/transformers/models/deberta/tokenization_deberta.py index 6bca0ed581..13bb8b4817 100644 --- a/src/transformers/models/deberta/tokenization_deberta.py +++ b/src/transformers/models/deberta/tokenization_deberta.py @@ -210,7 +210,7 @@ class DebertaTokenizer(GPT2Tokenizer): if token_ids_1 is None: return len(cls + token_ids_0 + sep) * [0] - return len(cls + token_ids_0 + sep + token_ids_1 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs): add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space) diff --git a/src/transformers/models/deberta/tokenization_deberta_fast.py b/src/transformers/models/deberta/tokenization_deberta_fast.py index 74c2e4aca2..62deff8b14 100644 --- a/src/transformers/models/deberta/tokenization_deberta_fast.py +++ b/src/transformers/models/deberta/tokenization_deberta_fast.py @@ -183,7 +183,7 @@ class DebertaTokenizerFast(GPT2TokenizerFast): sequence pair mask has the following format: ``` - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 | first sequence | second sequence | ``` @@ -203,4 +203,4 @@ class DebertaTokenizerFast(GPT2TokenizerFast): if token_ids_1 is None: return len(cls + token_ids_0 + sep) * [0] - return len(cls + token_ids_0 + sep + token_ids_1 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] diff --git a/tests/models/deberta/test_tokenization_deberta.py b/tests/models/deberta/test_tokenization_deberta.py index dcbef8eae5..ca6574bc31 100644 --- a/tests/models/deberta/test_tokenization_deberta.py +++ b/tests/models/deberta/test_tokenization_deberta.py @@ -88,6 +88,12 @@ class DebertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase): input_bpe_tokens = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19] self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) + def test_token_type_ids(self): + tokenizer = self.get_tokenizer() + tokd = tokenizer("Hello", "World") + expected_token_type_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1] + self.assertListEqual(tokd["token_type_ids"], expected_token_type_ids) + @slow def test_sequence_builders(self): tokenizer = self.tokenizer_class.from_pretrained("microsoft/deberta-base")