Make the big table creation/check platform independent (#8856)

2020-12-01 11:45:57 -05:00
parent d366228df1
commit c0df963ee1
6 changed files with 92 additions and 41 deletions
--- a/src/transformers/utils/dummy_tokenizers_objects.py
+++ b/src/transformers/utils/dummy_tokenizers_objects.py
@@ -164,6 +164,15 @@ class MobileBertTokenizerFast:
        requires_tokenizers(self)


+class MT5TokenizerFast:
+    def __init__(self, *args, **kwargs):
+        requires_tokenizers(self)
+
+    @classmethod
+    def from_pretrained(self, *args, **kwargs):
+        requires_tokenizers(self)
+
+
 class OpenAIGPTTokenizerFast:
    def __init__(self, *args, **kwargs):
        requires_tokenizers(self)