Revert "Fix gradient checkpointing + fp16 autocast for most models" (#24420)

Revert "Fix gradient checkpointing + fp16 autocast for most models (#24247)"

This reverts commit 285a48011d.

tests/models/sam/test_modeling_sam.py

@@ -421,12 +421,6 @@ class SamModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
     def test_training_gradient_checkpointing(self):
         pass
 
-    @unittest.skip(
-        reason="The model does not support GC + autocast + fp16: https://github.com/huggingface/transformers/pull/24247"
-    )
-    def test_training_gradient_checkpointing_autocast(self):
-        pass
-
     @unittest.skip(reason="SamModel has no base class and is not available in MODEL_MAPPING")
     def test_save_load_fast_init_from_base(self):
         pass