SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix FP16 and attention masks in FunnelTransformer (#7374)

Browse Source 
* Fix #7371

* Fix training

* Fix test values

* Apply the fix to TF as well
This commit is contained in:
Sylvain Gugger
2020-09-25 12:20:39 -04:00
committed by GitHub
parent 4e5b036bdd
commit ad39271ae8
3 changed files with 10 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_modeling_funnel.py
View File

@@ -428,16 +428,16 @@ class FunnelModelIntegrationTest(unittest.TestCase):
model = FunnelModel.from_pretrained("sgugger/funnel-random-tiny")
output = model(input_ids, token_type_ids=token_type_ids)[0].abs()
expected_output_sum = torch.tensor(2344.9023)
expected_output_mean = torch.tensor(0.8053)
expected_output_sum = torch.tensor(2344.8352)
expected_output_mean = torch.tensor(0.8052)
self.assertTrue(torch.allclose(output.sum(), expected_output_sum, atol=1e-4))
self.assertTrue(torch.allclose(output.mean(), expected_output_mean, atol=1e-4))
attention_mask = torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]])
output = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)[0].abs()
expected_output_sum = torch.tensor(2363.2178)
expected_output_mean = torch.tensor(0.8115)
expected_output_sum = torch.tensor(2343.8425)
expected_output_mean = torch.tensor(0.8049)
self.assertTrue(torch.allclose(output.sum(), expected_output_sum, atol=1e-4))
self.assertTrue(torch.allclose(output.mean(), expected_output_mean, atol=1e-4))
@@ -448,7 +448,7 @@ class FunnelModelIntegrationTest(unittest.TestCase):
inputs = tokenizer("Hello! I am the Funnel Transformer model.", return_tensors="pt")
output = model(**inputs)[0]
expected_output_sum = torch.tensor(235.7827)
expected_output_sum = torch.tensor(235.7246)
expected_output_mean = torch.tensor(0.0256)
self.assertTrue(torch.allclose(output.sum(), expected_output_sum, atol=1e-4))
self.assertTrue(torch.allclose(output.mean(), expected_output_mean, atol=1e-4))