From 0a3d0e02c5af20bfe9091038c4fd11fb79175546 Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Wed, 24 Jun 2020 09:14:50 -0700 Subject: [PATCH] Replace labels with -100 to skip loss calc (#4718) --- src/transformers/data/data_collator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py index d575f48d6c..acb0807af1 100644 --- a/src/transformers/data/data_collator.py +++ b/src/transformers/data/data_collator.py @@ -82,7 +82,9 @@ class DataCollatorForLanguageModeling: inputs, labels = self.mask_tokens(batch) return {"input_ids": inputs, "labels": labels} else: - return {"input_ids": batch, "labels": batch} + labels = batch.clone().detach() + labels[labels == self.tokenizer.pad_token_id] = -100 + return {"input_ids": batch, "labels": labels} def _tensorize_batch(self, examples: List[torch.Tensor]) -> torch.Tensor: length_of_first = examples[0].size(0)