SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Golden path

Browse Source
This commit is contained in:
jinoobaek-qz
2019-10-07 15:26:57 -07:00
committed by Lysandre Debut
parent 528d3f327b
commit bf34a252b8
1 changed files with 18 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
examples/run_lm_finetuning.py
View File

@@ -114,23 +114,25 @@ def _rotate_checkpoints(args, checkpoint_prefix, use_mtime=False):
# Check if we should delete older checkpoint(s) # Check if we should delete older checkpoint(s)
glob_checkpoints = glob.glob(os.path.join(args.output_dir, '{}-*'.format(checkpoint_prefix))) glob_checkpoints = glob.glob(os.path.join(args.output_dir, '{}-*'.format(checkpoint_prefix)))
if len(glob_checkpoints) > args.save_total_limit: if len(glob_checkpoints) <= args.save_total_limit:
checkpoints_sorted = [] return
for path in glob_checkpoints:
regex_match = re.match('.*{}-([0-9]+)'.format(checkpoint_prefix), path)
if regex_match and regex_match.groups():
if use_mtime:
checkpoints_sorted.append((os.path.getmtime(path), path))
else:
checkpoints_sorted.append((int(regex_match.groups()[0]), path))
checkpoints_sorted = sorted(checkpoints_sorted) checkpoints_sorted = []
checkpoints_sorted = [checkpoint[1] for checkpoint in checkpoints_sorted] for path in glob_checkpoints:
number_of_checkpoints_to_delete = max(0, len(checkpoints_sorted) - args.save_total_limit) regex_match = re.match('.*{}-([0-9]+)'.format(checkpoint_prefix), path)
checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete] if regex_match and regex_match.groups():
for checkpoint in checkpoints_to_be_deleted: if use_mtime:
logger.info("Deleting older checkpoint [{}] due to args.save_total_limit".format(checkpoint)) checkpoints_sorted.append((os.path.getmtime(path), path))
shutil.rmtree(checkpoint) else:
checkpoints_sorted.append((int(regex_match.groups()[0]), path))
checkpoints_sorted = sorted(checkpoints_sorted)
checkpoints_sorted = [checkpoint[1] for checkpoint in checkpoints_sorted]
number_of_checkpoints_to_delete = max(0, len(checkpoints_sorted) - args.save_total_limit)
checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete]
for checkpoint in checkpoints_to_be_deleted:
logger.info("Deleting older checkpoint [{}] due to args.save_total_limit".format(checkpoint))
shutil.rmtree(checkpoint)
def mask_tokens(inputs, tokenizer, args): def mask_tokens(inputs, tokenizer, args):