SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

small clean up in tests

Browse Source
This commit is contained in:
thomwolf
2019-04-17 14:52:12 +02:00
parent 23d4554ec0
commit 34ae5bf838
1 changed files with 0 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/tokenization_gpt2_test.py
View File

@@ -40,7 +40,6 @@ class GPT2TokenizationTest(unittest.TestCase):
merges_file = fp.name merges_file = fp.name
tokenizer = GPT2Tokenizer(vocab_file, merges_file, special_tokens=["<unk>", "<pad>"]) tokenizer = GPT2Tokenizer(vocab_file, merges_file, special_tokens=["<unk>", "<pad>"])
print("encoder", tokenizer.byte_encoder)
os.remove(vocab_file) os.remove(vocab_file)
os.remove(merges_file) os.remove(merges_file)