From 96e83506d1ddee8e19b07118668be73d175decb6 Mon Sep 17 00:00:00 2001 From: Ethan Perez Date: Fri, 29 Nov 2019 18:22:34 -0600 Subject: [PATCH] Always use SequentialSampler during evaluation When evaluating, shouldn't we always use the SequentialSampler instead of DistributedSampler? Evaluation only runs on 1 GPU no matter what, so if you use the DistributedSampler with N GPUs, I think you'll only evaluate on 1/N of the evaluation set. That's at least what I'm finding when I run an older/modified version of this repo. --- examples/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 59683c0668..32d807b3ad 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -216,7 +216,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) + eval_sampler = SequentialSampler(dataset) eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate