Check models used for common tests are small (#24824)

* First models

* Conditional DETR

* Treat DETR models, skip others

* Skip LayoutLMv2 as well

* Fix last tests

tests/models/bridgetower/test_modeling_bridgetower.py

@@ -62,12 +62,12 @@ class BridgeTowerTextModelTester:
         self,
         parent,
         hidden_act="gelu",
-        hidden_size=128,
+        hidden_size=64,
         initializer_factor=1,
         layer_norm_eps=1e-05,
         num_attention_heads=4,
         num_hidden_layers=2,
-        intermediate_size=256,
+        intermediate_size=128,
         tie_word_embeddings=False,
         output_hidden_states=False,
     ):
@@ -105,6 +105,7 @@ class BridgeTowerTextModelTester:
             intermediate_size=self.intermediate_size,
             tie_word_embeddings=self.tie_word_embeddings,
             output_hidden_states=self.output_hidden_states,
+            vocab_size=self.vocab_size,
         )
 
 
@@ -112,7 +113,7 @@ class BridgeTowerImageModelTester:
     def __init__(
         self,
         parent,
-        hidden_size=128,
+        hidden_size=64,
         initializer_factor=1,
         layer_norm_eps=1e-05,
         num_hidden_layers=2,
@@ -168,10 +169,10 @@ class BridgeTowerModelTester:
         init_layernorm_from_vision_encoder=False,
         contrastive_hidden_size=512,
         logit_scale_init_value=2.6592,
-        hidden_size=128,
+        hidden_size=64,
         num_hidden_layers=2,
         num_attention_heads=4,
-        intermediate_size=256,
+        intermediate_size=128,
     ):
         if text_kwargs is None:
             text_kwargs = {}
@@ -280,7 +281,10 @@ class BridgeTowerModelTester:
         result = model(input_ids, attention_mask=attention_mask, pixel_values=pixel_values, pixel_mask=pixel_mask)
         result = model(input_ids, attention_mask=attention_mask, pixel_values=pixel_values)
 
-        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.text_model_tester.seq_length, 50265))
+        self.parent.assertEqual(
+            result.logits.shape,
+            (self.batch_size, self.text_model_tester.seq_length, self.text_model_tester.vocab_size),
+        )
 
     def prepare_config_and_inputs_for_common(self):
         config_and_inputs = self.prepare_config_and_inputs()