Use Filelock to ensure distributed barriers

see context in https://github.com/huggingface/transformers/pull/4223
2020-05-14 11:58:32 -04:00
parent 015f7812ed
commit c547f15a17
6 changed files with 25 additions and 33 deletions
--- a/examples/multiple-choice/run_multiple_choice.py
+++ b/examples/multiple-choice/run_multiple_choice.py
@@ -159,7 +159,6 @@ def main():
            max_seq_length=data_args.max_seq_length,
            overwrite_cache=data_args.overwrite_cache,
            mode=Split.train,
-            local_rank=training_args.local_rank,
        )
        if training_args.do_train
        else None
@@ -172,7 +171,6 @@ def main():
            max_seq_length=data_args.max_seq_length,
            overwrite_cache=data_args.overwrite_cache,
            mode=Split.dev,
-            local_rank=training_args.local_rank,
        )
        if training_args.do_eval
        else None