SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix TF Causal LM models' returned logits (#15256)

Browse Source 
* Fix TF Causal LM models' returned logits

* Fix expected shape in the tests

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2022-02-01 12:04:07 +01:00
committed by GitHub
parent af5c3329d7
commit dc05dd539f
10 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/models/bert/modeling_tf_bert.py
View File

@@ -1542,9 +1542,9 @@ class TFBertLMHeadModel(TFBertPreTrainedModel, TFCausalLanguageModelingLoss):
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels=labels, logits=logits) loss = self.hf_compute_loss(labels=labels, logits=shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + outputs[2:] output = (logits,) + outputs[2:]

4
src/transformers/models/ctrl/modeling_tf_ctrl.py
View File

@@ -735,9 +735,9 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel, TFCausalLanguageModelingLoss):
loss = None loss = None
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels, logits) loss = self.hf_compute_loss(labels, shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + transformer_outputs[1:] output = (logits,) + transformer_outputs[1:]

4
src/transformers/models/gpt2/modeling_tf_gpt2.py
View File

@@ -949,9 +949,9 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel, TFCausalLanguageModelingLoss):
loss = None loss = None
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels, logits) loss = self.hf_compute_loss(labels, shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + transformer_outputs[1:] output = (logits,) + transformer_outputs[1:]

4
src/transformers/models/openai/modeling_tf_openai.py
View File

@@ -656,9 +656,9 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel, TFCausalLanguageModelin
loss = None loss = None
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels, logits) loss = self.hf_compute_loss(labels, shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + transformer_outputs[1:] output = (logits,) + transformer_outputs[1:]

4
src/transformers/models/rembert/modeling_tf_rembert.py
View File

@@ -1275,9 +1275,9 @@ class TFRemBertForCausalLM(TFRemBertPreTrainedModel, TFCausalLanguageModelingLos
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels=labels, logits=logits) loss = self.hf_compute_loss(labels=labels, logits=shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + outputs[2:] output = (logits,) + outputs[2:]

4
src/transformers/models/roberta/modeling_tf_roberta.py
View File

@@ -1310,9 +1310,9 @@ class TFRobertaForCausalLM(TFRobertaPreTrainedModel, TFCausalLanguageModelingLos
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels=labels, logits=logits) loss = self.hf_compute_loss(labels=labels, logits=shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + outputs[2:] output = (logits,) + outputs[2:]

4
src/transformers/models/roformer/modeling_tf_roformer.py
View File

@@ -1035,9 +1035,9 @@ class TFRoFormerForCausalLM(TFRoFormerPreTrainedModel, TFCausalLanguageModelingL
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels=labels, logits=logits) loss = self.hf_compute_loss(labels=labels, logits=shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + outputs[2:] output = (logits,) + outputs[2:]

4
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py
View File

@@ -1262,9 +1262,9 @@ class TF{{cookiecutter.camelcase_modelname}}ForCausalLM(TF{{cookiecutter.camelca
if inputs["labels"] is not None: if inputs["labels"] is not None:
# shift labels to the left and cut last logit token # shift labels to the left and cut last logit token
logits = logits[:, :-1] shifted_logits = logits[:, :-1]
labels = inputs["labels"][:, 1:] labels = inputs["labels"][:, 1:]
loss = self.hf_compute_loss(labels=labels, logits=logits) loss = self.hf_compute_loss(labels=labels, logits=shifted_logits)
if not inputs["return_dict"]: if not inputs["return_dict"]:
output = (logits,) + outputs[2:] output = (logits,) + outputs[2:]

2
tests/test_modeling_tf_encoder_decoder.py
View File

@@ -240,7 +240,7 @@ class TFEncoderDecoderMixin:
assert "loss" in outputs_encoder_decoder assert "loss" in outputs_encoder_decoder
batch_size, seq_len = decoder_input_ids.shape batch_size, seq_len = decoder_input_ids.shape
expected_shape = (batch_size, seq_len - 1, decoder_config.vocab_size) expected_shape = (batch_size, seq_len, decoder_config.vocab_size)
self.assertEqual(outputs_encoder_decoder["logits"].shape, expected_shape) self.assertEqual(outputs_encoder_decoder["logits"].shape, expected_shape)
self.assertEqual( self.assertEqual(
outputs_encoder_decoder["encoder_last_hidden_state"].shape, (input_ids.shape + (config.hidden_size,)) outputs_encoder_decoder["encoder_last_hidden_state"].shape, (input_ids.shape + (config.hidden_size,))

2
tests/test_modeling_tf_vision_encoder_decoder.py
View File

@@ -231,7 +231,7 @@ class TFVisionEncoderDecoderMixin:
self.assertIn("loss", outputs_encoder_decoder) self.assertIn("loss", outputs_encoder_decoder)
batch_size, seq_len = decoder_input_ids.shape batch_size, seq_len = decoder_input_ids.shape
expected_shape = (batch_size, seq_len - 1, decoder_config.vocab_size) expected_shape = (batch_size, seq_len, decoder_config.vocab_size)
self.assertEqual(outputs_encoder_decoder["logits"].shape, expected_shape) self.assertEqual(outputs_encoder_decoder["logits"].shape, expected_shape)
self.assertEqual(outputs_encoder_decoder["encoder_last_hidden_state"].shape[0], pixel_values.shape[0]) self.assertEqual(outputs_encoder_decoder["encoder_last_hidden_state"].shape[0], pixel_values.shape[0])
self.assertEqual(outputs_encoder_decoder["encoder_last_hidden_state"].shape[-1], config.hidden_size) self.assertEqual(outputs_encoder_decoder["encoder_last_hidden_state"].shape[-1], config.hidden_size)