SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix: verify jsonlines file in run_translation (#14660) (#14661)

Browse Source 
* fix: verify jsonl in run_translation (#14660)

* fix(run_translation.py): json/jsonl validation

Both json and jsonl are to be accepted as valid jsonlines file extension

* fix(run_translation.py): make black happy

* Ran make style
This commit is contained in:
Gaurang Tandon
2021-12-08 18:25:30 +00:00
committed by GitHub
parent cf36f4d7a8
commit 4ea19de80c
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
examples/pytorch/translation/run_translation.py
View File

@@ -216,12 +216,16 @@ class DataTrainingArguments:
elif self.source_lang is None or self.target_lang is None: elif self.source_lang is None or self.target_lang is None:
raise ValueError("Need to specify the source language and the target language.") raise ValueError("Need to specify the source language and the target language.")
# accepting both json and jsonl file extensions, as
# many jsonlines files actually have a .json extension
valid_extensions = ["json", "jsonl"]
if self.train_file is not None: if self.train_file is not None:
extension = self.train_file.split(".")[-1] extension = self.train_file.split(".")[-1]
assert extension == "json", "`train_file` should be a json file." assert extension in valid_extensions, "`train_file` should be a jsonlines file."
if self.validation_file is not None: if self.validation_file is not None:
extension = self.validation_file.split(".")[-1] extension = self.validation_file.split(".")[-1]
assert extension == "json", "`validation_file` should be a json file." assert extension in valid_extensions, "`validation_file` should be a jsonlines file."
if self.val_max_target_length is None: if self.val_max_target_length is None:
self.val_max_target_length = self.max_target_length self.val_max_target_length = self.max_target_length