[MPNet] Add slow to fast tokenizer converter (#9233)
* add converter * delet unnecessary comments
This commit is contained in:
committed by
GitHub
parent
f4432b7e01
commit
9a12b9696f
@@ -17,6 +17,7 @@
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from transformers import MPNetTokenizerFast
|
||||
from transformers.models.mpnet.tokenization_mpnet import VOCAB_FILES_NAMES, MPNetTokenizer
|
||||
from transformers.testing_utils import require_tokenizers, slow
|
||||
|
||||
@@ -27,7 +28,9 @@ from .test_tokenization_common import TokenizerTesterMixin
|
||||
class MPNetTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
|
||||
tokenizer_class = MPNetTokenizer
|
||||
test_rust_tokenizer = False
|
||||
rust_tokenizer_class = MPNetTokenizerFast
|
||||
test_rust_tokenizer = True
|
||||
space_between_special_tokens = True
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
Reference in New Issue
Block a user