SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add missing lang tokens in M2M100Tokenizer.get_vocab (#18416)

Browse Source
This commit is contained in:
Guillaume Klein
2022-10-25 15:18:24 +02:00
committed by GitHub
parent 0bd6d9340e
commit ab108a0e31
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/m2m_100/tokenization_m2m_100.py
View File

@@ -280,7 +280,7 @@ class M2M100Tokenizer(PreTrainedTokenizer):
return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens
def get_vocab(self) -> Dict: def get_vocab(self) -> Dict:
vocab = self.encoder.copy() vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
vocab.update(self.added_tokens_encoder) vocab.update(self.added_tokens_encoder)
return vocab return vocab

5
tests/models/m2m_100/test_tokenization_m2m_100.py
View File

@@ -89,7 +89,7 @@ class M2M100TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
self.assertEqual(vocab_keys[0], "</s>") self.assertEqual(vocab_keys[0], "</s>")
self.assertEqual(vocab_keys[1], "<unk>") self.assertEqual(vocab_keys[1], "<unk>")
self.assertEqual(vocab_keys[-1], "<s>") self.assertEqual(vocab_keys[-1], "<s>")
self.assertEqual(len(vocab_keys), 10) self.assertEqual(len(vocab_keys), 110)
def test_vocab_size(self): def test_vocab_size(self):
self.assertEqual(self.get_tokenizer().vocab_size, 117) self.assertEqual(self.get_tokenizer().vocab_size, 117)
@@ -160,6 +160,9 @@ class M2M100TokenizerIntegrationTest(unittest.TestCase):
self.assertEqual(self.tokenizer.get_lang_id("ro"), 128076) self.assertEqual(self.tokenizer.get_lang_id("ro"), 128076)
self.assertEqual(self.tokenizer.get_lang_id("mr"), 128063) self.assertEqual(self.tokenizer.get_lang_id("mr"), 128063)
def test_get_vocab(self):
self.assertIn(self.tokenizer.get_lang_token("en"), self.tokenizer.get_vocab())
def test_tokenizer_batch_encode_plus(self): def test_tokenizer_batch_encode_plus(self):
self.tokenizer.src_lang = "en" self.tokenizer.src_lang = "en"
ids = self.tokenizer.batch_encode_plus(self.src_text).input_ids[0] ids = self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]