Use Filelock to ensure distributed barriers

see context in https://github.com/huggingface/transformers/pull/4223

examples/multiple-choice/utils_multiple_choice.py

@@ -26,6 +26,7 @@ from enum import Enum
 from typing import List, Optional
 
 import tqdm
+from filelock import FileLock
 
 from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available
 
@@ -77,7 +78,6 @@ class Split(Enum):
 if is_torch_available():
     import torch
     from torch.utils.data.dataset import Dataset
-    from transformers import torch_distributed_zero_first
 
     class MultipleChoiceDataset(Dataset):
         """
@@ -95,7 +95,6 @@ if is_torch_available():
             max_seq_length: Optional[int] = None,
             overwrite_cache=False,
             mode: Split = Split.train,
-            local_rank=-1,
         ):
             processor = processors[task]()
 
@@ -103,9 +102,11 @@ if is_torch_available():
                 data_dir,
                 "cached_{}_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length), task,),
             )
-            with torch_distributed_zero_first(local_rank):
-                # Make sure only the first process in distributed training processes the dataset,
-                # and the others will use the cache.
+
+            # Make sure only the first process in distributed training processes the dataset,
+            # and the others will use the cache.
+            lock_path = cached_features_file + ".lock"
+            with FileLock(lock_path):
 
                 if os.path.exists(cached_features_file) and not overwrite_cache:
                     logger.info(f"Loading features from cached file {cached_features_file}")
@@ -130,9 +131,8 @@ if is_torch_available():
                         pad_token=tokenizer.pad_token_id,
                         pad_token_segment_id=tokenizer.pad_token_type_id,
                     )
-                    if local_rank in [-1, 0]:
-                        logger.info("Saving features into cached file %s", cached_features_file)
-                        torch.save(self.features, cached_features_file)
+                    logger.info("Saving features into cached file %s", cached_features_file)
+                    torch.save(self.features, cached_features_file)
 
         def __len__(self):
             return len(self.features)