From d5239bab5bd87d8f719aff7827edee63aad51f60 Mon Sep 17 00:00:00 2001 From: Mikel Penagarikano Date: Wed, 5 Apr 2023 15:36:04 +0200 Subject: [PATCH] Sync preprocesses before loading the processor at run_speech_recognition_ctc.py (#21926) * Update run_speech_recognition_ctc.py Make sure all processes wait until data is saved before loading the processor from the output_dit * Make sure all processes wait until data is saved before loading the processor from the output_dit * Update run_speech_recognition_ctc.py * Update run_speech_recognition_seq2seq.py --- .../run_speech_recognition_ctc.py | 13 ++++++++----- .../run_speech_recognition_seq2seq.py | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py index a6c71e5e70..a702e460aa 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py @@ -673,11 +673,14 @@ def main(): return metrics # Now save everything to be able to create a single processor later - if is_main_process(training_args.local_rank): - # save feature extractor, tokenizer and config - feature_extractor.save_pretrained(training_args.output_dir) - tokenizer.save_pretrained(training_args.output_dir) - config.save_pretrained(training_args.output_dir) + # make sure all processes wait until data is saved + with training_args.main_process_first(): + # only the main process saves them + if is_main_process(training_args.local_rank): + # save feature extractor, tokenizer and config + feature_extractor.save_pretrained(training_args.output_dir) + tokenizer.save_pretrained(training_args.output_dir) + config.save_pretrained(training_args.output_dir) try: processor = AutoProcessor.from_pretrained(training_args.output_dir) diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py index 86213c47f5..79826f0d9d 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py @@ -506,11 +506,14 @@ def main(): return {"wer": wer} # 9. Create a single speech processor - if is_main_process(training_args.local_rank): - # save feature extractor, tokenizer and config - feature_extractor.save_pretrained(training_args.output_dir) - tokenizer.save_pretrained(training_args.output_dir) - config.save_pretrained(training_args.output_dir) + # make sure all processes wait until data is saved + with training_args.main_process_first(): + # only the main process saves them + if is_main_process(training_args.local_rank): + # save feature extractor, tokenizer and config + feature_extractor.save_pretrained(training_args.output_dir) + tokenizer.save_pretrained(training_args.output_dir) + config.save_pretrained(training_args.output_dir) processor = AutoProcessor.from_pretrained(training_args.output_dir)