fix python 2.7 imports

2019-02-11 10:35:36 +01:00
parent 8197eb9f10
commit 2071a9b86e
3 changed files with 15 additions and 2 deletions
--- a/tests/tokenization_openai_test.py
+++ b/tests/tokenization_openai_test.py
@@ -32,7 +32,7 @@ class OpenAIGPTTokenizationTest(unittest.TestCase):
                 "low</w>", "lowest</w>", "newer</w>", "wider</w>"]
        vocab_tokens = dict(zip(vocab, range(len(vocab))))
        merges = ["#version: 0.2", "l o", "lo w", "e r</w>", ""]
-        with open("/tmp/openai_tokenizer_vocab_test.json", "w", encoding='utf-8') as fp:
+        with open("/tmp/openai_tokenizer_vocab_test.json", "wb") as fp:
            json.dump(vocab_tokens, fp)
            vocab_file = fp.name
        with open("/tmp/openai_tokenizer_merges_test.txt", "w", encoding='utf-8') as fp: