SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

examples: only use keep_linebreaks when reading TXT files (#13320)

Browse Source 
* examples: only use keep_linebreaks when reading TXT files for all CLM examples

* examples: only use keep_linebreaks when reading TXT files for all CLM examples

* examples: only use keep_linebreaks when reading TXT files for all CLM examples
This commit is contained in:
Stefan Schweter
2021-08-28 16:22:29 +02:00
committed by GitHub
parent b6f332ecaf
commit 4046e66e40
4 changed files with 22 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/tensorflow/language-modeling/run_clm.py
View File

@@ -187,7 +187,7 @@ class DataTrainingArguments:
},
)
keep_linebreaks: bool = field(
default=True, metadata={"help": "Whether to keep line breaks when using CSV/JSON/TXT files or not."}
default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
)
def __post_init__(self):
@@ -321,6 +321,7 @@ def main():
)
else:
data_files = {}
dataset_args = {}
if data_args.train_file is not None:
data_files["train"] = data_args.train_file
if data_args.validation_file is not None:
@@ -328,7 +329,8 @@ def main():
extension = data_args.train_file.split(".")[-1]
if extension == "txt":
extension = "text"
raw_datasets = load_dataset(extension, keep_linebreaks=data_args.keep_linebreaks, data_files=data_files)
dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
raw_datasets = load_dataset(extension, data_files=data_files, **dataset_args)
# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
# https://huggingface.co/docs/datasets/loading_datasets.html.
# endregion