SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[cuda ext tests] fixing tests (#11619)

Browse Source 
* fixing tests

* cleanup
This commit is contained in:
Stas Bekman
2021-05-06 13:35:28 -07:00
committed by GitHub
parent 44c5621db0
commit 619200cc42
3 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/deepspeed/test_deepspeed.py
View File

@@ -318,9 +318,10 @@ class TrainerIntegrationDeepSpeed(TestCasePlus, TrainerIntegrationCommon):
yes_grad_accum_b = yes_grad_accum_trainer.model.b.item()
self.assertNotEqual(yes_grad_accum_a, a)
# training with half the batch size but accumulation steps as 2 should give the same weights
self.assertEqual(no_grad_accum_a, yes_grad_accum_a)
self.assertEqual(no_grad_accum_b, yes_grad_accum_b)
# training with half the batch size but accumulation steps as 2 should give the same
# weights, but sometimes get a slight difference still of 1e-6
self.assertAlmostEqual(no_grad_accum_a, yes_grad_accum_a, places=5)
self.assertAlmostEqual(no_grad_accum_b, yes_grad_accum_b, places=5)
# see the note above how to get identical loss on a small bs
self.assertAlmostEqual(no_grad_accum_loss, yes_grad_accum_loss, places=5)