[MPNet] Add slow to fast tokenizer converter (#9233)

* add converter

* delet unnecessary comments
This commit is contained in:
Patrick von Platen
2020-12-21 15:41:34 +01:00
committed by GitHub
parent f4432b7e01
commit 9a12b9696f
2 changed files with 44 additions and 25 deletions

View File

@@ -17,6 +17,7 @@
import os
import unittest
from transformers import MPNetTokenizerFast
from transformers.models.mpnet.tokenization_mpnet import VOCAB_FILES_NAMES, MPNetTokenizer
from transformers.testing_utils import require_tokenizers, slow
@@ -27,7 +28,9 @@ from .test_tokenization_common import TokenizerTesterMixin
class MPNetTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = MPNetTokenizer
test_rust_tokenizer = False
rust_tokenizer_class = MPNetTokenizerFast
test_rust_tokenizer = True
space_between_special_tokens = True
def setUp(self):
super().setUp()