CI with num_hidden_layers=2 🚀🚀🚀 (#25266)

* CI with layers=2

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/pegasus/test_modeling_flax_pegasus.py

@@ -52,7 +52,7 @@ class FlaxPegasusModelTester:
         use_labels=False,
         vocab_size=99,
         hidden_size=32,
-        num_hidden_layers=5,
+        num_hidden_layers=2,
         num_attention_heads=4,
         intermediate_size=37,
         hidden_dropout_prob=0.1,

tests/models/pegasus/test_modeling_pegasus.py

@@ -371,7 +371,7 @@ class PegasusStandaloneDecoderModelTester:
         use_labels=True,
         decoder_start_token_id=2,
         decoder_ffn_dim=32,
-        decoder_layers=4,
+        decoder_layers=2,
         encoder_attention_heads=4,
         decoder_attention_heads=4,
         max_position_embeddings=30,