SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Close open files to suppress ResourceWarning (#11240)

Browse Source 
Co-authored-by: Sudharsan Thirumalai <sudharsan.t@sprinklr.com>
This commit is contained in:
Sudharsan S T
2021-04-14 20:01:04 +05:30
committed by GitHub
parent 7fe5aaa8b0
commit f25444cb22
3 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/legacy/seq2seq/run_distributed_eval.py
View File

@@ -204,7 +204,8 @@ def run_generate():
save_json(preds, save_path) save_json(preds, save_path)
return return
tgt_file = Path(args.data_dir).joinpath(args.type_path + ".target") tgt_file = Path(args.data_dir).joinpath(args.type_path + ".target")
labels = [x.rstrip() for x in open(tgt_file).readlines()][: len(preds)] with open(tgt_file) as f:
labels = [x.rstrip() for x in f.readlines()][: len(preds)]
# Calculate metrics, save metrics, and save _generations.txt # Calculate metrics, save metrics, and save _generations.txt
calc_bleu = "translation" in args.task calc_bleu = "translation" in args.task

3
examples/research_projects/seq2seq-distillation/run_eval.py
View File

@@ -115,7 +115,8 @@ def run_generate(verbose=True):
parsed_args = parse_numeric_n_bool_cl_kwargs(rest) parsed_args = parse_numeric_n_bool_cl_kwargs(rest)
if parsed_args and verbose: if parsed_args and verbose:
print(f"parsed the following generate kwargs: {parsed_args}") print(f"parsed the following generate kwargs: {parsed_args}")
examples = [" " + x.rstrip() if "t5" in args.model_name else x.rstrip() for x in open(args.input_path).readlines()] with open(args.input_path) as f:
examples = [" " + x.rstrip() if "t5" in args.model_name else x.rstrip() for x in f.readlines()]
if args.n_obs > 0: if args.n_obs > 0:
examples = examples[: args.n_obs] examples = examples[: args.n_obs]
Path(args.save_path).parent.mkdir(exist_ok=True) Path(args.save_path).parent.mkdir(exist_ok=True)

3
src/transformers/convert_slow_tokenizer.py
View File

@@ -305,7 +305,8 @@ class SpmConverter(Converter):
from .utils import sentencepiece_model_pb2 as model_pb2 from .utils import sentencepiece_model_pb2 as model_pb2
m = model_pb2.ModelProto() m = model_pb2.ModelProto()
m.ParseFromString(open(self.original_tokenizer.vocab_file, "rb").read()) with open(self.original_tokenizer.vocab_file, "rb") as f:
m.ParseFromString(f.read())
self.proto = m self.proto = m
def vocab(self, proto): def vocab(self, proto):