Fix last models for common tests that are too big. (#25058)

* Fix last models for common tests that are too big.

* Remove print statement

tests/models/perceiver/test_modeling_perceiver.py

@@ -79,6 +79,7 @@ class PerceiverModelTester:
         nchunks=20,
         num_latents=10,
         d_latents=20,
+        d_model=64,
         num_blocks=1,
         num_self_attends_per_block=2,
         num_self_attention_heads=1,
@@ -108,6 +109,7 @@ class PerceiverModelTester:
         self.nchunks = nchunks
         self.num_latents = num_latents
         self.d_latents = d_latents
+        self.d_model = d_model
         self.num_blocks = num_blocks
         self.num_self_attends_per_block = num_self_attends_per_block
         self.num_self_attention_heads = num_self_attention_heads
@@ -181,6 +183,7 @@ class PerceiverModelTester:
         return PerceiverConfig(
             num_latents=self.num_latents,
             d_latents=self.d_latents,
+            d_model=self.d_model,
             qk_channels=self.d_latents,
             v_channels=self.d_latents,
             num_blocks=self.num_blocks,
@@ -200,6 +203,8 @@ class PerceiverModelTester:
             audio_samples_per_frame=self.audio_samples_per_frame,
             samples_per_patch=self.samples_per_patch,
             num_labels=self.num_labels,
+            output_num_channels=32,
+            _label_trainable_num_channels=16,
         )
 
     def get_pipeline_config(self):
@@ -784,10 +789,6 @@ class PerceiverModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCas
 
                     loss.backward()
 
-    @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests.")
-    def test_model_is_small(self):
-        pass
-
     @require_torch_multi_gpu
     @unittest.skip(
         reason=(