SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix DeBERTa integration tests (#7729)

Browse Source
This commit is contained in:
Lysandre Debut
2020-10-16 08:49:13 +02:00
committed by GitHub
parent 2255c2c7a0
commit 52c9e84285
2 changed files with 1 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/modeling_deberta.py
View File

@@ -491,7 +491,7 @@ class DisentangledSelfAttention(torch.nn.Module):
self.in_proj = torch.nn.Linear(config.hidden_size, self.all_head_size * 3, bias=False) self.in_proj = torch.nn.Linear(config.hidden_size, self.all_head_size * 3, bias=False)
self.q_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float)) self.q_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
self.v_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float)) self.v_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
self.pos_att_type = config.pos_att_type self.pos_att_type = config.pos_att_type if config.pos_att_type is not None else []
self.relative_attention = getattr(config, "relative_attention", False) self.relative_attention = getattr(config, "relative_attention", False)
self.talking_head = getattr(config, "talking_head", False) self.talking_head = getattr(config, "talking_head", False)

15
tests/test_modeling_deberta.py
View File

@@ -247,7 +247,6 @@ class DebertaModelIntegrationTest(unittest.TestCase):
np.random.seed(0) np.random.seed(0)
torch.manual_seed(0) torch.manual_seed(0)
torch.cuda.manual_seed_all(0) torch.cuda.manual_seed_all(0)
DebertaModel.base_model_prefix = "bert"
model = DebertaModel.from_pretrained("microsoft/deberta-base") model = DebertaModel.from_pretrained("microsoft/deberta-base")
input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]])
@@ -257,17 +256,3 @@ class DebertaModelIntegrationTest(unittest.TestCase):
[[[-0.0218, -0.6641, -0.3665], [-0.3907, -0.4716, -0.6640], [0.7461, 1.2570, -0.9063]]] [[[-0.0218, -0.6641, -0.3665], [-0.3907, -0.4716, -0.6640], [0.7461, 1.2570, -0.9063]]]
) )
self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4), f"{output[:, :3, :3]}") self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4), f"{output[:, :3, :3]}")
@slow
def test_inference_classification_head(self):
random.seed(0)
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)
model = DebertaForSequenceClassification.from_pretrained("microsoft/deberta-base")
input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]])
output = model(input_ids)[0]
expected_shape = torch.Size((1, 2))
self.assertEqual(output.shape, expected_shape)
expected_tensor = torch.tensor([[0.0884, -0.1047]])
self.assertTrue(torch.allclose(output, expected_tensor, atol=1e-4), f"{output}")