SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Implement Fast Tokenization for Deberta (#11387)

Browse Source
This commit is contained in:
Shubham Sanghavi
2021-04-30 07:08:15 -05:00
committed by GitHub
parent db9dd09cf9
commit 30ede8994e
9 changed files with 271 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/test_tokenization_deberta.py
View File

@@ -18,7 +18,7 @@ import json
import os
import unittest
from transformers import DebertaTokenizer
from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow
@@ -28,7 +28,8 @@ from .test_tokenization_common import TokenizerTesterMixin
class DebertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = DebertaTokenizer
test_rust_tokenizer = False
test_rust_tokenizer = True
rust_tokenizer_class = DebertaTokenizerFast
def setUp(self):
super().setUp()