Implement Fast Tokenization for Deberta (#11387)

2021-04-30 07:08:15 -05:00
parent db9dd09cf9
commit 30ede8994e
9 changed files with 271 additions and 5 deletions
--- a/src/transformers/utils/dummy_tokenizers_objects.py
+++ b/src/transformers/utils/dummy_tokenizers_objects.py
@@ -56,6 +56,15 @@ class ConvBertTokenizerFast:
        requires_backends(self, ["tokenizers"])


+class DebertaTokenizerFast:
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["tokenizers"])
+
+    @classmethod
+    def from_pretrained(self, *args, **kwargs):
+        requires_backends(self, ["tokenizers"])
+
+
 class DistilBertTokenizerFast:
    def __init__(self, *args, **kwargs):
        requires_backends(self, ["tokenizers"])