SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

disable_ngram_loss fix for prophetnet (#8554)

Browse Source 
* `disable_ngram_loss` fix for prophetnet

* add changes documentation

* fix _compute_loss to use mean reduction and -100 to masked tokens & remove unnecessary arguments

* mean label smoothing loss

* small refactor

* fix test

Co-authored-by: patrickvonplaten <patrick.v.platen@gmail.com>
This commit is contained in:
Zhylko Dima
2020-11-19 19:18:07 +01:00
committed by GitHub
parent 0603564e93
commit ca0109bd68
2 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/transformers/models/prophetnet/modeling_prophetnet.py
View File

@@ -1793,8 +1793,8 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel):
encoder_attentions=outputs.encoder_attentions, encoder_attentions=outputs.encoder_attentions,
) )
def _compute_loss(self, logits, labels): def _compute_loss(self, logits, labels, ignore_index=-100):
expend_targets = labels.new_zeros(self.config.ngram, labels.size(0), labels.size(1)).fill_(self.padding_idx) expend_targets = labels.new_zeros(self.config.ngram, labels.size(0), labels.size(1)).fill_(ignore_index)
for i in range(self.config.ngram): for i in range(self.config.ngram):
if i > 0 and self.disable_ngram_loss: if i > 0 and self.disable_ngram_loss:
@@ -1807,13 +1807,13 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel):
dtype=torch.float32, dtype=torch.float32,
) )
loss = F.nll_loss(lprobs, expend_targets.view(-1), reduction="sum") loss = F.nll_loss(lprobs, expend_targets.view(-1), reduction="mean")
if self.config.eps > 0.0: if self.config.eps > 0.0:
smooth_loss = -lprobs.sum(dim=-1, keepdim=True) smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
non_pad_mask = expend_targets.ne(self.padding_idx).view(-1) non_masked_tokens = expend_targets.ne(ignore_index).view(-1)
smooth_loss = smooth_loss[non_pad_mask] smooth_loss = smooth_loss[non_masked_tokens]
smooth_loss = smooth_loss.sum() smooth_loss = smooth_loss.mean()
eps_i = self.config.eps / lprobs.size(-1) eps_i = self.config.eps / lprobs.size(-1)
loss = (1.0 - self.config.eps) * loss + eps_i * smooth_loss loss = (1.0 - self.config.eps) * loss + eps_i * smooth_loss
@@ -2010,8 +2010,8 @@ class ProphetNetForCausalLM(ProphetNetPreTrainedModel):
cross_attentions=outputs.cross_attentions, cross_attentions=outputs.cross_attentions,
) )
def _compute_loss(self, logits, labels): def _compute_loss(self, logits, labels, ignore_index=-100):
expend_targets = labels.new_zeros(self.config.ngram, labels.size(0), labels.size(1)).fill_(self.padding_idx) expend_targets = labels.new_zeros(self.config.ngram, labels.size(0), labels.size(1)).fill_(ignore_index)
for i in range(self.config.ngram): for i in range(self.config.ngram):
if i > 0 and self.disable_ngram_loss: if i > 0 and self.disable_ngram_loss:
@@ -2024,13 +2024,13 @@ class ProphetNetForCausalLM(ProphetNetPreTrainedModel):
dtype=torch.float32, dtype=torch.float32,
) )
loss = F.nll_loss(lprobs, expend_targets.view(-1), reduction="sum") loss = F.nll_loss(lprobs, expend_targets.view(-1), reduction="mean")
if self.config.eps > 0.0: if self.config.eps > 0.0:
smooth_loss = -lprobs.sum(dim=-1, keepdim=True) smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
non_pad_mask = expend_targets.ne(self.padding_idx).view(-1) non_masked_tokens = expend_targets.ne(ignore_index).view(-1)
smooth_loss = smooth_loss[non_pad_mask] smooth_loss = smooth_loss[non_masked_tokens]
smooth_loss = smooth_loss.sum() smooth_loss = smooth_loss.mean()
eps_i = self.config.eps / lprobs.size(-1) eps_i = self.config.eps / lprobs.size(-1)
loss = (1.0 - self.config.eps) * loss + eps_i * smooth_loss loss = (1.0 - self.config.eps) * loss + eps_i * smooth_loss

2
tests/test_modeling_prophetnet.py
View File

@@ -417,7 +417,7 @@ class ProphetNetModelTester:
decoder_attention_mask=decoder_attention_mask, decoder_attention_mask=decoder_attention_mask,
labels=lm_labels, labels=lm_labels,
) )
self.parent.assertTrue(torch.allclose(result.loss, torch.tensor(128.2925, device=torch_device), atol=1e-3)) self.parent.assertTrue(torch.allclose(result.loss, torch.tensor(4.5819, device=torch_device), atol=1e-3))
expected_logit_slice = torch.tensor( expected_logit_slice = torch.tensor(
[-0.1565, 0.0418, 0.1207, 0.0030, 0.0665, 0.0467, 0.0412], device=torch_device [-0.1565, 0.0418, 0.1207, 0.0030, 0.0665, 0.0467, 0.0412], device=torch_device