[cleanup] test_tokenization_common.py (#4390)

2020-05-19 10:46:55 -04:00
parent 8f1d047148
commit 07dd7c2fd8
13 changed files with 62 additions and 98 deletions
--- a/tests/test_tokenization_bert.py
+++ b/tests/test_tokenization_bert.py
@@ -59,9 +59,6 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
        with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))

-    def get_tokenizer(self, **kwargs):
-        return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
-
    def get_rust_tokenizer(self, **kwargs):
        return BertTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)