[Tests] Improve test_attention_outputs (#20701)

* Improve tests

* Improve TF tests

* Apply suggestion

* Fix test

Co-authored-by: Niels Rogge <nielsrogge@Nielss-MacBook-Pro.local>

tests/test_modeling_common.py

@@ -554,6 +554,9 @@ class ModelTesterMixin:
             loss.backward()
 
     def test_attention_outputs(self):
+        if not self.has_attentions:
+            self.skipTest(reason="Model does not output attentions")
+
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
         config.return_dict = True