SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

save_pretrained: mkdir(exist_ok=True) (#5258)

Browse Source 
* all save_pretrained methods mkdir if not os.path.exists
This commit is contained in:
Sam Shleifer
2020-06-28 14:53:47 -04:00
committed by GitHub
parent 12dfbd4f7a
commit 45e26125de
16 changed files with 17 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/transformers/tokenization_utils_base.py
View File

@@ -1343,9 +1343,10 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
This method make sure the full tokenizer can then be re-loaded using the
:func:`~transformers.PreTrainedTokenizer.from_pretrained` class method.
"""
if not os.path.isdir(save_directory):
logger.error("Saving directory ({}) should be a directory".format(save_directory))
if os.path.isfile(save_directory):
logger.error("Provided path ({}) should be a directory, not a file".format(save_directory))
return
os.makedirs(save_directory, exist_ok=True)
special_tokens_map_file = os.path.join(save_directory, SPECIAL_TOKENS_MAP_FILE)
added_tokens_file = os.path.join(save_directory, ADDED_TOKENS_FILE)