SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[s2s] fix label_smoothed_nll_loss (#6344)

Browse Source
This commit is contained in:
Suraj Patil
2020-08-08 13:51:12 +05:30
committed by GitHub
parent 99f73bcc71
commit 9bed355449
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/seq2seq/utils.py
View File

@@ -29,17 +29,15 @@ def label_smoothed_nll_loss(lprobs, target, epsilon, ignore_index=-100):
pad_mask = target.eq(ignore_index) pad_mask = target.eq(ignore_index)
nll_loss.masked_fill_(pad_mask, 0.0) nll_loss.masked_fill_(pad_mask, 0.0)
smooth_loss.masked_fill_(pad_mask, 0.0) smooth_loss.masked_fill_(pad_mask, 0.0)
bs = pad_mask.long().sum()
else: else:
nll_loss = nll_loss.squeeze(-1) nll_loss = nll_loss.squeeze(-1)
smooth_loss = smooth_loss.squeeze(-1) smooth_loss = smooth_loss.squeeze(-1)
bs = lprobs.shape[0]
nll_loss = nll_loss.sum() # mean()? Scared to break other math. nll_loss = nll_loss.sum() # mean()? Scared to break other math.
smooth_loss = smooth_loss.sum() smooth_loss = smooth_loss.sum()
eps_i = epsilon / lprobs.size(-1) eps_i = epsilon / lprobs.size(-1)
loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss
return loss / bs, nll_loss / bs return loss, nll_loss
def encode_line(tokenizer, line, max_length, pad_to_max_length=True, return_tensors="pt"): def encode_line(tokenizer, line, max_length, pad_to_max_length=True, return_tensors="pt"):