SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[examples/summarization] deal with None in data records (#14816)

Browse Source 
* [examples/summarization] deal with None in data records

* rewrite to use a simpler (slower) variant
This commit is contained in:
Stas Bekman
2021-12-21 09:17:28 -08:00
committed by GitHub
parent c075fb7855
commit 033c3ed95a
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
examples/pytorch/summarization/run_summarization.py
View File

@@ -436,8 +436,14 @@ def main():
) )
def preprocess_function(examples): def preprocess_function(examples):
inputs = examples[text_column]
targets = examples[summary_column] # remove pairs where at least one record is None
inputs, targets = [], []
for i in range(len(examples[text_column])):
if examples[text_column][i] is not None and examples[summary_column][i] is not None:
inputs.append(examples[text_column][i])
targets.append(examples[summary_column][i])
inputs = [prefix + inp for inp in inputs] inputs = [prefix + inp for inp in inputs]
model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, padding=padding, truncation=True) model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, padding=padding, truncation=True)