SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

map only on one process (#13810)

Browse Source
This commit is contained in:
Patrick von Platen
2021-09-30 18:52:53 +02:00
committed by GitHub
parent 9a9805fccf
commit 44eb8bdeea
9 changed files with 142 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
examples/pytorch/token-classification/run_ner_no_trainer.py
View File

@@ -403,12 +403,13 @@ def main():
tokenized_inputs["labels"] = labels
return tokenized_inputs
processed_raw_datasets = raw_datasets.map(
tokenize_and_align_labels,
batched=True,
remove_columns=raw_datasets["train"].column_names,
desc="Running tokenizer on dataset",
)
with accelerator.main_process_first():
processed_raw_datasets = raw_datasets.map(
tokenize_and_align_labels,
batched=True,
remove_columns=raw_datasets["train"].column_names,
desc="Running tokenizer on dataset",
)
train_dataset = processed_raw_datasets["train"]
eval_dataset = processed_raw_datasets["validation"]