SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

python2 doesn't spark joy

Browse Source
This commit is contained in:
thomwolf
2019-08-30 13:22:43 +02:00
parent 5dd7b677ad
commit ce5ef4b35d
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytorch_transformers/tests/tokenization_gpt2_test.py
View File

@@ -42,7 +42,7 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester):
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file'])
self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file'])
with open(self.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens))
fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.merges_file, "w", encoding="utf-8") as fp:
fp.write("\n".join(merges))

2
pytorch_transformers/tests/tokenization_roberta_test.py
View File

@@ -41,7 +41,7 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester):
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file'])
self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file'])
with open(self.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens))
fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.merges_file, "w", encoding="utf-8") as fp:
fp.write("\n".join(merges))