SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[tests] Light cleanup of tempfile in tests/

Browse Source
This commit is contained in:
Julien Chaumond
2020-04-23 17:16:55 -04:00
parent f54dc3f4d5
commit f39217a5ec
1 changed files with 12 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tests/test_tokenization_common.py
View File

@@ -107,15 +107,14 @@ class TokenizerTesterMixin:
before_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
with tempfile.TemporaryDirectory() as tmpdirname:
tokenizer.save_pretrained(tmpdirname)
tokenizer = self.tokenizer_class.from_pretrained(tmpdirname)
tokenizer.save_pretrained(self.tmpdirname)
tokenizer = self.tokenizer_class.from_pretrained(self.tmpdirname)
after_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
self.assertListEqual(before_tokens, after_tokens)
self.assertEqual(tokenizer.max_len, 42)
tokenizer = self.tokenizer_class.from_pretrained(tmpdirname, max_len=43)
tokenizer = self.tokenizer_class.from_pretrained(self.tmpdirname, max_len=43)
self.assertEqual(tokenizer.max_len, 43)
def test_pickle_tokenizer(self):
@@ -125,9 +124,7 @@ class TokenizerTesterMixin:
text = "Munich and Berlin are nice cities"
subwords = tokenizer.tokenize(text)
with tempfile.TemporaryDirectory() as tmpdirname:
filename = os.path.join(tmpdirname, "tokenizer.bin")
filename = os.path.join(self.tmpdirname, "tokenizer.bin")
with open(filename, "wb") as handle:
pickle.dump(tokenizer, handle)