[tests] Stricter generate + compilation test -- no recompilations allowed (#37629)

* tmp commit

* stricter compilation test

* trigger tests

* rm todo

tests/models/janus/test_modeling_janus.py

@@ -335,6 +335,10 @@ class JanusVisionText2TextModelTest(ModelTesterMixin, GenerationTesterMixin, uni
                         else:
                             pass
 
+    @unittest.skip("There are recompilations in Janus")  # TODO (joao, raushan): fix me
+    def test_generate_compile_model_forward(self):
+        pass
+
 
 class JanusVQModelTester:
     def __init__(