set find_unused_parameters=True in DDP

2019-06-18 12:13:14 +02:00
parent a59abedfb5
commit d82e5deeb1
2 changed files with 8 additions and 4 deletions
--- a/README.md
+++ b/README.md
@@ -1468,12 +1468,13 @@ python -m torch.distributed.launch --nproc_per_node=8 \
  --do_lower_case \
  --train_file $SQUAD_DIR/train-v1.1.json \
  --predict_file $SQUAD_DIR/dev-v1.1.json \
-  --train_batch_size 12 \
  --learning_rate 3e-5 \
-  --num_train_epochs 2.0 \
+  --num_train_epochs 2 \
  --max_seq_length 384 \
  --doc_stride 128 \
-  --output_dir /tmp/debug_squad/
+  --output_dir /tmp/debug_squad/ \
+  --train_batch_size 24 \
+  --gradient_accumulation_steps 2
 ```

 ## Notebooks