[cleanup] test_tokenization_common.py (#4390)

2020-05-19 10:46:55 -04:00
parent 8f1d047148
commit 07dd7c2fd8
13 changed files with 62 additions and 98 deletions
--- a/tests/test_tokenization_openai.py
+++ b/tests/test_tokenization_openai.py
@@ -64,13 +64,8 @@ class OpenAIGPTTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
        with open(self.merges_file, "w") as fp:
            fp.write("\n".join(merges))

-    def get_tokenizer(self, **kwargs):
-        return OpenAIGPTTokenizer.from_pretrained(self.tmpdirname, **kwargs)
-
    def get_input_output_texts(self):
-        input_text = "lower newer"
-        output_text = "lower newer"
-        return input_text, output_text
+        return "lower newer", "lower newer"

    def test_full_tokenizer(self):
        tokenizer = OpenAIGPTTokenizer(self.vocab_file, self.merges_file)