SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[T5] Fix speed degradation bug t5 (#10496)

Browse Source 
* fix speed degradation bug t5

* fix for all models

* fix code quality
This commit is contained in:
Patrick von Platen
2021-03-03 12:42:41 +03:00
committed by GitHub
parent 5dc303e281
commit 2d2ed2cc18
9 changed files with 30 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/models/bart/modeling_bart.py
View File

@@ -319,7 +319,9 @@ class BartEncoderLayer(nn.Module):
hidden_states = residual + hidden_states
hidden_states = self.final_layer_norm(hidden_states)
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

4
src/transformers/models/blenderbot/modeling_blenderbot.py
View File

@@ -322,7 +322,9 @@ class BlenderbotEncoderLayer(nn.Module):
hidden_states = F.dropout(hidden_states, p=self.dropout, training=self.training)
hidden_states = residual + hidden_states
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

4
src/transformers/models/blenderbot_small/modeling_blenderbot_small.py
View File

@@ -320,7 +320,9 @@ class BlenderbotSmallEncoderLayer(nn.Module):
hidden_states = residual + hidden_states
hidden_states = self.final_layer_norm(hidden_states)
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

4
src/transformers/models/led/modeling_led.py
View File

@@ -925,7 +925,9 @@ class LEDEncoderLayer(nn.Module):
hidden_states = residual + hidden_states
hidden_states = self.final_layer_norm(hidden_states)
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
return (hidden_states,) + attn_outputs[1:]

4
src/transformers/models/marian/modeling_marian.py
View File

@@ -337,7 +337,9 @@ class MarianEncoderLayer(nn.Module):
hidden_states = residual + hidden_states
hidden_states = self.final_layer_norm(hidden_states)
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

4
src/transformers/models/mbart/modeling_mbart.py
View File

@@ -326,7 +326,9 @@ class MBartEncoderLayer(nn.Module):
hidden_states = F.dropout(hidden_states, p=self.dropout, training=self.training)
hidden_states = residual + hidden_states
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

4
src/transformers/models/pegasus/modeling_pegasus.py
View File

@@ -337,7 +337,9 @@ class PegasusEncoderLayer(nn.Module):
hidden_states = F.dropout(hidden_states, p=self.dropout, training=self.training)
hidden_states = residual + hidden_states
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (
torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

11
src/transformers/models/t5/modeling_t5.py
View File

@@ -643,7 +643,7 @@ class T5Block(nn.Module):
attention_outputs = self_attention_outputs[2:] # Keep self-attention outputs and relative position weights
# clamp inf values to enable fp16 training
if torch.isinf(hidden_states).any():
if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
@@ -668,7 +668,9 @@ class T5Block(nn.Module):
output_attentions=output_attentions,
)
hidden_states = cross_attention_outputs[0]
if torch.isinf(hidden_states).any():
# clamp inf values to enable fp16 training
if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
@@ -681,9 +683,12 @@ class T5Block(nn.Module):
# Apply Feed Forward layer
hidden_states = self.layer[-1](hidden_states)
if torch.isinf(hidden_states).any():
# clamp inf values to enable fp16 training
if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
outputs = (hidden_states,)
outputs = outputs + (present_key_value_state,) + attention_outputs

2
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_{{cookiecutter.lowercase_modelname}}.py
View File

@@ -1824,7 +1824,7 @@ class {{cookiecutter.camelcase_modelname}}EncoderLayer(nn.Module):
hidden_states = residual + hidden_states
hidden_states = self.final_layer_norm(hidden_states)
if torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any():
if hidden_states.dtype == torch.float16 and (torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()):
clamp_value = torch.finfo(hidden_states.dtype).max - 1000
hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)