SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix gradient checkpointing + fp16 autocast for most models (#24247)

Browse Source 
* fix gc bug

* continue PoC on OPT

* fixes

* 🤯

* fix tests

* remove pytest.mark

* fixup

* forward contrib credits from discussions

* forward contrib credits from discussions

* reverting changes on untouched files.

---------

Co-authored-by: zhaoqf123 <zhaoqf123@users.noreply.github.com>
Co-authored-by: 7eu7d7 <7eu7d7@users.noreply.github.com>
This commit is contained in:
Younes Belkada
2023-06-21 17:04:59 +02:00
committed by GitHub
parent 1815d1865e
commit 285a48011d
179 changed files with 836 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/models/flava/test_modeling_flava.py
View File

@@ -185,6 +185,12 @@ class FlavaImageModelTest(ModelTesterMixin, unittest.TestCase):
# FLAVA does not use inputs_embeds
pass
@unittest.skip(
reason="The model does not support GC + autocast + fp16: https://github.com/huggingface/transformers/pull/24247"
)
def test_training_gradient_checkpointing_autocast(self):
pass
def test_model_common_attributes(self):
config, _ = self.model_tester.prepare_config_and_inputs_for_common()
@@ -462,6 +468,12 @@ class FlavaTextModelTest(ModelTesterMixin, unittest.TestCase):
# FLAVA does not use inputs_embeds
pass
@unittest.skip(
reason="The model does not support GC + autocast + fp16: https://github.com/huggingface/transformers/pull/24247"
)
def test_training_gradient_checkpointing_autocast(self):
pass
# skip this test as FlavaTextModel has no base class and is
# not available in MODEL_MAPPING
def test_save_load_fast_init_from_base(self):
@@ -624,6 +636,12 @@ class FlavaMultimodalModelTest(ModelTesterMixin, unittest.TestCase):
def test_save_load_fast_init_to_base(self):
pass
@unittest.skip(
reason="The model does not support GC + autocast + fp16: https://github.com/huggingface/transformers/pull/24247"
)
def test_training_gradient_checkpointing_autocast(self):
pass
@slow
def test_model_from_pretrained(self):
for model_name in FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
@@ -731,6 +749,12 @@ class FlavaImageCodebookTest(ModelTesterMixin, unittest.TestCase):
def test_save_load_fast_init_to_base(self):
pass
@unittest.skip(
reason="The model does not support GC + autocast + fp16: https://github.com/huggingface/transformers/pull/24247"
)
def test_training_gradient_checkpointing_autocast(self):
pass
@slow
def test_model_from_pretrained(self):
for model_name in FLAVA_CODEBOOK_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
@@ -1156,6 +1180,12 @@ class FlavaForPreTrainingTest(FlavaModelTest):
class_for_tester = FlavaForPreTrainingTester
test_torchscript = False
@unittest.skip(
reason="The model does not support GC + autocast + fp16: https://github.com/huggingface/transformers/pull/24247"
)
def test_training_gradient_checkpointing_autocast(self):
pass
# We will verify our results on an image of cute cats
def prepare_img():