SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[run_clm.py] fix getting extention

Browse Source
This commit is contained in:
Suraj Patil
2021-02-03 20:14:42 +05:30
committed by GitHub
parent 5442a11f5f
commit bca0dd5ee3
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/language-modeling/run_clm.py
View File

@@ -227,7 +227,11 @@ def main():
data_files["train"] = data_args.train_file data_files["train"] = data_args.train_file
if data_args.validation_file is not None: if data_args.validation_file is not None:
data_files["validation"] = data_args.validation_file data_files["validation"] = data_args.validation_file
extension = data_args.train_file.split(".")[-1] extension = (
data_args.train_file.split(".")[-1]
if data_args.train_file is not None
else data_args.validation_file.split(".")[-1]
)
if extension == "txt": if extension == "txt":
extension = "text" extension = "text"
datasets = load_dataset(extension, data_files=data_files) datasets = load_dataset(extension, data_files=data_files)