Replace (TF)CommonTestCases for modeling with a mixin.

I suspect the wrapper classes were created in order to prevent the
abstract base class (TF)CommonModelTester from being included in test
discovery and running, because that would fail.

I solved this by replacing the abstract base class with a mixin.

Code changes are just de-indenting and automatic reformattings
performed by black to use the extra line space.

templates/adding_a_new_model/tests/test_modeling_tf_xxx.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import XxxConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -32,7 +34,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (

templates/adding_a_new_model/tests/test_modeling_xxx.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -34,7 +36,7 @@ if is_torch_available():
 
 
 @require_torch
-class XxxModelTest(CommonTestCases.CommonModelTester):
+class XxxModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (XxxModel, XxxForMaskedLM, XxxForQuestionAnswering, XxxForSequenceClassification, XxxForTokenClassification)

tests/test_modeling_albert.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -33,7 +35,7 @@ if is_torch_available():
 
 
 @require_torch
-class AlbertModelTest(CommonTestCases.CommonModelTester):
+class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else ()
 

tests/test_modeling_bert.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, floats_tensor, ids_tensor
+from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -37,7 +39,7 @@ if is_torch_available():
 
 
 @require_torch
-class BertModelTest(CommonTestCases.CommonModelTester):
+class BertModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (

tests/test_modeling_ctrl.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -25,7 +27,7 @@ if is_torch_available():
 
 
 @require_torch
-class CTRLModelTest(CommonTestCases.CommonModelTester):
+class CTRLModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (CTRLModel, CTRLLMHeadModel) if is_torch_available() else ()
     test_pruning = False

tests/test_modeling_distilbert.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import require_torch, torch_device
 
 
@@ -33,7 +35,7 @@ if is_torch_available():
 
 
 @require_torch
-class DistilBertModelTest(CommonTestCases.CommonModelTester):
+class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification)

tests/test_modeling_gpt2.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -32,7 +34,7 @@ if is_torch_available():
 
 
 @require_torch
-class GPT2ModelTest(CommonTestCases.CommonModelTester):
+class GPT2ModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) if is_torch_available() else ()
 

tests/test_modeling_openai.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -32,7 +34,7 @@ if is_torch_available():
 
 
 @require_torch
-class OpenAIGPTModelTest(CommonTestCases.CommonModelTester):
+class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) if is_torch_available() else ()

tests/test_modeling_roberta.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -37,7 +37,7 @@ if is_torch_available():
 
 
 @require_torch
-class RobertaModelTest(CommonTestCases.CommonModelTester):
+class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (RobertaForMaskedLM, RobertaModel) if is_torch_available() else ()
 

tests/test_modeling_t5.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow
 
 
@@ -27,7 +29,7 @@ if is_torch_available():
 
 
 @require_torch
-class T5ModelTest(CommonTestCases.CommonModelTester):
+class T5ModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (T5Model, T5WithLMHeadModel) if is_torch_available() else ()
     test_pruning = False

tests/test_modeling_tf_albert.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import AlbertConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -31,7 +33,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (TFAlbertModel, TFAlbertForMaskedLM, TFAlbertForSequenceClassification) if is_tf_available() else ()

tests/test_modeling_tf_bert.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import BertConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -36,7 +38,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFBertModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (

tests/test_modeling_tf_common.py

@@ -20,7 +20,6 @@ import random
 import shutil
 import sys
 import tempfile
-import unittest
 
 from transformers import is_tf_available, is_torch_available
 
@@ -59,307 +58,300 @@ def _config_zero_init(config):
     return configs_no_init
 
 
-class TFCommonTestCases:
+@require_tf
-    @require_tf
+class TFModelTesterMixin:
-    class TFCommonModelTester(unittest.TestCase):
 
-        model_tester = None
+    model_tester = None
-        all_model_classes = ()
+    all_model_classes = ()
-        test_torchscript = True
+    test_torchscript = True
-        test_pruning = True
+    test_pruning = True
-        test_resize_embeddings = True
+    test_resize_embeddings = True
-        is_encoder_decoder = False
+    is_encoder_decoder = False
 
-        def test_initialization(self):
+    def test_initialization(self):
-            pass
+        pass
-            # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
 
-            # configs_no_init = _config_zero_init(config)
+        # configs_no_init = _config_zero_init(config)
-            # for model_class in self.all_model_classes:
+        # for model_class in self.all_model_classes:
-            #     model = model_class(config=configs_no_init)
+        #     model = model_class(config=configs_no_init)
-            #     for name, param in model.named_parameters():
+        #     for name, param in model.named_parameters():
-            #         if param.requires_grad:
+        #         if param.requires_grad:
-            #             self.assertIn(param.data.mean().item(), [0.0, 1.0],
+        #             self.assertIn(param.data.mean().item(), [0.0, 1.0],
-            #             msg="Parameter {} of model {} seems not properly initialized".format(name, model_class))
+        #             msg="Parameter {} of model {} seems not properly initialized".format(name, model_class))
 
-        def test_save_load(self):
+    def test_save_load(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
 
-            for model_class in self.all_model_classes:
+        for model_class in self.all_model_classes:
-                model = model_class(config)
+            model = model_class(config)
-                outputs = model(inputs_dict)
+            outputs = model(inputs_dict)
 
-                with TemporaryDirectory() as tmpdirname:
+            with TemporaryDirectory() as tmpdirname:
-                    model.save_pretrained(tmpdirname)
+                model.save_pretrained(tmpdirname)
-                    model = model_class.from_pretrained(tmpdirname)
+                model = model_class.from_pretrained(tmpdirname)
-                    after_outputs = model(inputs_dict)
+                after_outputs = model(inputs_dict)
 
-                    # Make sure we don't have nans
+                # Make sure we don't have nans
-                    out_1 = after_outputs[0].numpy()
+                out_1 = after_outputs[0].numpy()
-                    out_2 = outputs[0].numpy()
+                out_2 = outputs[0].numpy()
-                    out_1 = out_1[~np.isnan(out_1)]
-                    out_2 = out_2[~np.isnan(out_2)]
-                    max_diff = np.amax(np.abs(out_1 - out_2))
-                    self.assertLessEqual(max_diff, 1e-5)
-
-        def test_pt_tf_model_equivalence(self):
-            if not is_torch_available():
-                return
-
-            import torch
-            import transformers
-
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            for model_class in self.all_model_classes:
-                pt_model_class_name = model_class.__name__[2:]  # Skip the "TF" at the beggining
-                pt_model_class = getattr(transformers, pt_model_class_name)
-
-                config.output_hidden_states = True
-                tf_model = model_class(config)
-                pt_model = pt_model_class(config)
-
-                # Check we can load pt model in tf and vice-versa with model => model functions
-                tf_model = transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=inputs_dict)
-                pt_model = transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model)
-
-                # Check predictions on first output (logits/hidden-states) are close enought given low-level computational differences
-                pt_model.eval()
-                pt_inputs_dict = dict(
-                    (name, torch.from_numpy(key.numpy()).to(torch.long)) for name, key in inputs_dict.items()
-                )
-                with torch.no_grad():
-                    pto = pt_model(**pt_inputs_dict)
-                tfo = tf_model(inputs_dict, training=False)
-                tf_hidden_states = tfo[0].numpy()
-                pt_hidden_states = pto[0].numpy()
-                tf_hidden_states[np.isnan(tf_hidden_states)] = 0
-                pt_hidden_states[np.isnan(pt_hidden_states)] = 0
-                max_diff = np.amax(np.abs(tf_hidden_states - pt_hidden_states))
-                self.assertLessEqual(max_diff, 2e-2)
-
-                # Check we can load pt model in tf and vice-versa with checkpoint => model functions
-                with TemporaryDirectory() as tmpdirname:
-                    pt_checkpoint_path = os.path.join(tmpdirname, "pt_model.bin")
-                    torch.save(pt_model.state_dict(), pt_checkpoint_path)
-                    tf_model = transformers.load_pytorch_checkpoint_in_tf2_model(tf_model, pt_checkpoint_path)
-
-                    tf_checkpoint_path = os.path.join(tmpdirname, "tf_model.h5")
-                    tf_model.save_weights(tf_checkpoint_path)
-                    pt_model = transformers.load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path)
-
-                # Check predictions on first output (logits/hidden-states) are close enought given low-level computational differences
-                pt_model.eval()
-                pt_inputs_dict = dict(
-                    (name, torch.from_numpy(key.numpy()).to(torch.long)) for name, key in inputs_dict.items()
-                )
-                with torch.no_grad():
-                    pto = pt_model(**pt_inputs_dict)
-                tfo = tf_model(inputs_dict)
-                tfo = tfo[0].numpy()
-                pto = pto[0].numpy()
-                tfo[np.isnan(tfo)] = 0
-                pto[np.isnan(pto)] = 0
-                max_diff = np.amax(np.abs(tfo - pto))
-                self.assertLessEqual(max_diff, 2e-2)
-
-        def test_compile_tf_model(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            if self.is_encoder_decoder:
-                input_ids = {
-                    "decoder_input_ids": tf.keras.Input(
-                        batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"
-                    ),
-                    "encoder_input_ids": tf.keras.Input(
-                        batch_shape=(2, 2000), name="encoder_input_ids", dtype="int32"
-                    ),
-                }
-            else:
-                input_ids = tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32")
-            optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
-            loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-            metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
-
-            for model_class in self.all_model_classes:
-                # Prepare our model
-                model = model_class(config)
-
-                # Let's load it from the disk to be sure we can use pretrained weights
-                with TemporaryDirectory() as tmpdirname:
-                    outputs = model(inputs_dict)  # build the model
-                    model.save_pretrained(tmpdirname)
-                    model = model_class.from_pretrained(tmpdirname)
-
-                outputs_dict = model(input_ids)
-                hidden_states = outputs_dict[0]
-
-                # Add a dense layer on top to test intetgration with other keras modules
-                outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
-
-                # Compile extended model
-                extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
-                extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
-
-        def test_keyword_and_dict_args(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            for model_class in self.all_model_classes:
-                model = model_class(config)
-                outputs_dict = model(inputs_dict)
-
-                inputs_keywords = copy.deepcopy(inputs_dict)
-                input_ids = inputs_keywords.pop(
-                    "input_ids" if not self.is_encoder_decoder else "decoder_input_ids", None
-                )
-                outputs_keywords = model(input_ids, **inputs_keywords)
-
-                output_dict = outputs_dict[0].numpy()
-                output_keywords = outputs_keywords[0].numpy()
-
-                self.assertLess(np.sum(np.abs(output_dict - output_keywords)), 1e-6)
-
-        def test_attention_outputs(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            decoder_seq_length = (
-                self.model_tester.decoder_seq_length
-                if hasattr(self.model_tester, "decoder_seq_length")
-                else self.model_tester.seq_length
-            )
-            encoder_seq_length = (
-                self.model_tester.encoder_seq_length
-                if hasattr(self.model_tester, "encoder_seq_length")
-                else self.model_tester.seq_length
-            )
-            decoder_key_length = (
-                self.model_tester.key_length if hasattr(self.model_tester, "key_length") else decoder_seq_length
-            )
-            encoder_key_length = (
-                self.model_tester.key_length if hasattr(self.model_tester, "key_length") else encoder_seq_length
-            )
-
-            for model_class in self.all_model_classes:
-                config.output_attentions = True
-                config.output_hidden_states = False
-                model = model_class(config)
-                outputs = model(inputs_dict)
-                attentions = [t.numpy() for t in outputs[-1]]
-                self.assertEqual(model.config.output_attentions, True)
-                self.assertEqual(model.config.output_hidden_states, False)
-                self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)
-                self.assertListEqual(
-                    list(attentions[0].shape[-3:]),
-                    [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
-                )
-                out_len = len(outputs)
-
-                if self.is_encoder_decoder:
-                    self.assertEqual(out_len % 2, 0)
-                    decoder_attentions = outputs[(out_len // 2) - 1]
-                    self.assertEqual(model.config.output_attentions, True)
-                    self.assertEqual(model.config.output_hidden_states, False)
-                    self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
-                    self.assertListEqual(
-                        list(decoder_attentions[0].shape[-3:]),
-                        [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
-                    )
-
-                # Check attention is always last and order is fine
-                config.output_attentions = True
-                config.output_hidden_states = True
-                model = model_class(config)
-                outputs = model(inputs_dict)
-                self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs))
-                self.assertEqual(model.config.output_attentions, True)
-                self.assertEqual(model.config.output_hidden_states, True)
-
-                attentions = [t.numpy() for t in outputs[-1]]
-                self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)
-                self.assertListEqual(
-                    list(attentions[0].shape[-3:]),
-                    [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
-                )
-
-        def test_hidden_states_output(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            for model_class in self.all_model_classes:
-                config.output_hidden_states = True
-                config.output_attentions = False
-                model = model_class(config)
-                outputs = model(inputs_dict)
-                hidden_states = [t.numpy() for t in outputs[-1]]
-                self.assertEqual(model.config.output_attentions, False)
-                self.assertEqual(model.config.output_hidden_states, True)
-                self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1)
-                self.assertListEqual(
-                    list(hidden_states[0].shape[-2:]), [self.model_tester.seq_length, self.model_tester.hidden_size]
-                )
-
-        def test_model_common_attributes(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            for model_class in self.all_model_classes:
-                model = model_class(config)
-                assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer)
-                x = model.get_output_embeddings()
-                assert x is None or isinstance(x, tf.keras.layers.Layer)
-
-        def test_determinism(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-            for model_class in self.all_model_classes:
-                model = model_class(config)
-                first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0]
-                out_1 = first.numpy()
-                out_2 = second.numpy()
                 out_1 = out_1[~np.isnan(out_1)]
                 out_2 = out_2[~np.isnan(out_2)]
                 max_diff = np.amax(np.abs(out_1 - out_2))
                 self.assertLessEqual(max_diff, 1e-5)
 
-        def _get_embeds(self, wte, input_ids):
+    def test_pt_tf_model_equivalence(self):
-            # ^^ In our TF models, the input_embeddings can take slightly different forms,
+        if not is_torch_available():
-            # so we try a few of them.
+            return
-            # We used to fall back to just synthetically creating a dummy tensor of ones:
+
+        import torch
+        import transformers
+
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        for model_class in self.all_model_classes:
+            pt_model_class_name = model_class.__name__[2:]  # Skip the "TF" at the beggining
+            pt_model_class = getattr(transformers, pt_model_class_name)
+
+            config.output_hidden_states = True
+            tf_model = model_class(config)
+            pt_model = pt_model_class(config)
+
+            # Check we can load pt model in tf and vice-versa with model => model functions
+            tf_model = transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=inputs_dict)
+            pt_model = transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model)
+
+            # Check predictions on first output (logits/hidden-states) are close enought given low-level computational differences
+            pt_model.eval()
+            pt_inputs_dict = dict(
+                (name, torch.from_numpy(key.numpy()).to(torch.long)) for name, key in inputs_dict.items()
+            )
+            with torch.no_grad():
+                pto = pt_model(**pt_inputs_dict)
+            tfo = tf_model(inputs_dict, training=False)
+            tf_hidden_states = tfo[0].numpy()
+            pt_hidden_states = pto[0].numpy()
+            tf_hidden_states[np.isnan(tf_hidden_states)] = 0
+            pt_hidden_states[np.isnan(pt_hidden_states)] = 0
+            max_diff = np.amax(np.abs(tf_hidden_states - pt_hidden_states))
+            self.assertLessEqual(max_diff, 2e-2)
+
+            # Check we can load pt model in tf and vice-versa with checkpoint => model functions
+            with TemporaryDirectory() as tmpdirname:
+                pt_checkpoint_path = os.path.join(tmpdirname, "pt_model.bin")
+                torch.save(pt_model.state_dict(), pt_checkpoint_path)
+                tf_model = transformers.load_pytorch_checkpoint_in_tf2_model(tf_model, pt_checkpoint_path)
+
+                tf_checkpoint_path = os.path.join(tmpdirname, "tf_model.h5")
+                tf_model.save_weights(tf_checkpoint_path)
+                pt_model = transformers.load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path)
+
+            # Check predictions on first output (logits/hidden-states) are close enought given low-level computational differences
+            pt_model.eval()
+            pt_inputs_dict = dict(
+                (name, torch.from_numpy(key.numpy()).to(torch.long)) for name, key in inputs_dict.items()
+            )
+            with torch.no_grad():
+                pto = pt_model(**pt_inputs_dict)
+            tfo = tf_model(inputs_dict)
+            tfo = tfo[0].numpy()
+            pto = pto[0].numpy()
+            tfo[np.isnan(tfo)] = 0
+            pto[np.isnan(pto)] = 0
+            max_diff = np.amax(np.abs(tfo - pto))
+            self.assertLessEqual(max_diff, 2e-2)
+
+    def test_compile_tf_model(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        if self.is_encoder_decoder:
+            input_ids = {
+                "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
+                "encoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="encoder_input_ids", dtype="int32"),
+            }
+        else:
+            input_ids = tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32")
+        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
+        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
+
+        for model_class in self.all_model_classes:
+            # Prepare our model
+            model = model_class(config)
+
+            # Let's load it from the disk to be sure we can use pretrained weights
+            with TemporaryDirectory() as tmpdirname:
+                outputs = model(inputs_dict)  # build the model
+                model.save_pretrained(tmpdirname)
+                model = model_class.from_pretrained(tmpdirname)
+
+            outputs_dict = model(input_ids)
+            hidden_states = outputs_dict[0]
+
+            # Add a dense layer on top to test intetgration with other keras modules
+            outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
+
+            # Compile extended model
+            extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
+            extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
+
+    def test_keyword_and_dict_args(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        for model_class in self.all_model_classes:
+            model = model_class(config)
+            outputs_dict = model(inputs_dict)
+
+            inputs_keywords = copy.deepcopy(inputs_dict)
+            input_ids = inputs_keywords.pop("input_ids" if not self.is_encoder_decoder else "decoder_input_ids", None)
+            outputs_keywords = model(input_ids, **inputs_keywords)
+
+            output_dict = outputs_dict[0].numpy()
+            output_keywords = outputs_keywords[0].numpy()
+
+            self.assertLess(np.sum(np.abs(output_dict - output_keywords)), 1e-6)
+
+    def test_attention_outputs(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        decoder_seq_length = (
+            self.model_tester.decoder_seq_length
+            if hasattr(self.model_tester, "decoder_seq_length")
+            else self.model_tester.seq_length
+        )
+        encoder_seq_length = (
+            self.model_tester.encoder_seq_length
+            if hasattr(self.model_tester, "encoder_seq_length")
+            else self.model_tester.seq_length
+        )
+        decoder_key_length = (
+            self.model_tester.key_length if hasattr(self.model_tester, "key_length") else decoder_seq_length
+        )
+        encoder_key_length = (
+            self.model_tester.key_length if hasattr(self.model_tester, "key_length") else encoder_seq_length
+        )
+
+        for model_class in self.all_model_classes:
+            config.output_attentions = True
+            config.output_hidden_states = False
+            model = model_class(config)
+            outputs = model(inputs_dict)
+            attentions = [t.numpy() for t in outputs[-1]]
+            self.assertEqual(model.config.output_attentions, True)
+            self.assertEqual(model.config.output_hidden_states, False)
+            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)
+            self.assertListEqual(
+                list(attentions[0].shape[-3:]),
+                [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
+            )
+            out_len = len(outputs)
+
+            if self.is_encoder_decoder:
+                self.assertEqual(out_len % 2, 0)
+                decoder_attentions = outputs[(out_len // 2) - 1]
+                self.assertEqual(model.config.output_attentions, True)
+                self.assertEqual(model.config.output_hidden_states, False)
+                self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
+                self.assertListEqual(
+                    list(decoder_attentions[0].shape[-3:]),
+                    [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
+                )
+
+            # Check attention is always last and order is fine
+            config.output_attentions = True
+            config.output_hidden_states = True
+            model = model_class(config)
+            outputs = model(inputs_dict)
+            self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs))
+            self.assertEqual(model.config.output_attentions, True)
+            self.assertEqual(model.config.output_hidden_states, True)
+
+            attentions = [t.numpy() for t in outputs[-1]]
+            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)
+            self.assertListEqual(
+                list(attentions[0].shape[-3:]),
+                [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
+            )
+
+    def test_hidden_states_output(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        for model_class in self.all_model_classes:
+            config.output_hidden_states = True
+            config.output_attentions = False
+            model = model_class(config)
+            outputs = model(inputs_dict)
+            hidden_states = [t.numpy() for t in outputs[-1]]
+            self.assertEqual(model.config.output_attentions, False)
+            self.assertEqual(model.config.output_hidden_states, True)
+            self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1)
+            self.assertListEqual(
+                list(hidden_states[0].shape[-2:]), [self.model_tester.seq_length, self.model_tester.hidden_size]
+            )
+
+    def test_model_common_attributes(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        for model_class in self.all_model_classes:
+            model = model_class(config)
+            assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer)
+            x = model.get_output_embeddings()
+            assert x is None or isinstance(x, tf.keras.layers.Layer)
+
+    def test_determinism(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        for model_class in self.all_model_classes:
+            model = model_class(config)
+            first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0]
+            out_1 = first.numpy()
+            out_2 = second.numpy()
+            out_1 = out_1[~np.isnan(out_1)]
+            out_2 = out_2[~np.isnan(out_2)]
+            max_diff = np.amax(np.abs(out_1 - out_2))
+            self.assertLessEqual(max_diff, 1e-5)
+
+    def _get_embeds(self, wte, input_ids):
+        # ^^ In our TF models, the input_embeddings can take slightly different forms,
+        # so we try a few of them.
+        # We used to fall back to just synthetically creating a dummy tensor of ones:
+        try:
+            x = wte(input_ids, mode="embedding")
+        except Exception:
             try:
-                x = wte(input_ids, mode="embedding")
+                x = wte([input_ids], mode="embedding")
             except Exception:
                 try:
-                    x = wte([input_ids], mode="embedding")
+                    x = wte([input_ids, None, None, None], mode="embedding")
                 except Exception:
-                    try:
+                    if hasattr(self.model_tester, "embedding_size"):
-                        x = wte([input_ids, None, None, None], mode="embedding")
+                        x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32)
-                    except Exception:
+                    else:
-                        if hasattr(self.model_tester, "embedding_size"):
+                        x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
-                            x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32)
+        return x
-                        else:
-                            x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32)
-            return x
 
-        def test_inputs_embeds(self):
+    def test_inputs_embeds(self):
-            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        if not self.is_encoder_decoder:
+            input_ids = inputs_dict["input_ids"]
+            del inputs_dict["input_ids"]
+        else:
+            encoder_input_ids = inputs_dict["encoder_input_ids"]
+            decoder_input_ids = inputs_dict["decoder_input_ids"]
+            del inputs_dict["encoder_input_ids"]
+            del inputs_dict["decoder_input_ids"]
+
+        for model_class in self.all_model_classes:
+            model = model_class(config)
+
+            wte = model.get_input_embeddings()
             if not self.is_encoder_decoder:
-                input_ids = inputs_dict["input_ids"]
+                inputs_dict["inputs_embeds"] = self._get_embeds(wte, input_ids)
-                del inputs_dict["input_ids"]
             else:
-                encoder_input_ids = inputs_dict["encoder_input_ids"]
+                inputs_dict["encoder_inputs_embeds"] = self._get_embeds(wte, encoder_input_ids)
-                decoder_input_ids = inputs_dict["decoder_input_ids"]
+                inputs_dict["decoder_inputs_embeds"] = self._get_embeds(wte, decoder_input_ids)
-                del inputs_dict["encoder_input_ids"]
-                del inputs_dict["decoder_input_ids"]
 
-            for model_class in self.all_model_classes:
+            outputs = model(inputs_dict)
-                model = model_class(config)
-
-                wte = model.get_input_embeddings()
-                if not self.is_encoder_decoder:
-                    inputs_dict["inputs_embeds"] = self._get_embeds(wte, input_ids)
-                else:
-                    inputs_dict["encoder_inputs_embeds"] = self._get_embeds(wte, encoder_input_ids)
-                    inputs_dict["decoder_inputs_embeds"] = self._get_embeds(wte, decoder_input_ids)
-
-                outputs = model(inputs_dict)
 
 
 def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None):

tests/test_modeling_tf_ctrl.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import CTRLConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -26,7 +28,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFCTRLModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (TFCTRLModel, TFCTRLLMHeadModel) if is_tf_available() else ()
 

tests/test_modeling_tf_distilbert.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import DistilBertConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import require_tf
 
 
@@ -31,7 +33,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (

tests/test_modeling_tf_gpt2.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import GPT2Config, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -32,7 +34,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel) if is_tf_available() else ()
     # all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel) if is_tf_available() else ()

tests/test_modeling_tf_openai_gpt.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import OpenAIGPTConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -32,7 +34,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFOpenAIGPTModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel) if is_tf_available() else ()

tests/test_modeling_tf_roberta.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import RobertaConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -36,7 +36,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification) if is_tf_available() else ()

tests/test_modeling_tf_t5.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import T5Config, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -26,7 +28,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFT5ModelTest(TFModelTesterMixin, unittest.TestCase):
 
     is_encoder_decoder = True
     all_model_classes = (TFT5Model, TFT5WithLMHeadModel) if is_tf_available() else ()

tests/test_modeling_tf_transfo_xl.py

@@ -15,11 +15,12 @@
 from __future__ import absolute_import, division, print_function
 
 import random
+import unittest
 
 from transformers import TransfoXLConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -33,7 +34,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFTransfoXLModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (TFTransfoXLModel, TFTransfoXLLMHeadModel) if is_tf_available() else ()
     test_pruning = False

tests/test_modeling_tf_xlm.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -34,7 +36,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple)

tests/test_modeling_tf_xlnet.py

@@ -15,11 +15,12 @@
 from __future__ import absolute_import, division, print_function
 
 import random
+import unittest
 
 from transformers import XLNetConfig, is_tf_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFCommonTestCases, ids_tensor
+from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_tf, slow
 
 
@@ -37,7 +38,7 @@ if is_tf_available():
 
 
 @require_tf
-class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester):
+class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (

tests/test_modeling_transfo_xl.py

@@ -15,11 +15,12 @@
 from __future__ import absolute_import, division, print_function
 
 import random
+import unittest
 
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -30,7 +31,7 @@ if is_torch_available():
 
 
 @require_torch
-class TransfoXLModelTest(CommonTestCases.CommonModelTester):
+class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) if is_torch_available() else ()
     test_pruning = False

tests/test_modeling_xlm.py

@@ -14,10 +14,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function
 
+import unittest
+
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -34,7 +36,7 @@ if is_torch_available():
 
 
 @require_torch
-class XLMModelTest(CommonTestCases.CommonModelTester):
+class XLMModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (

tests/test_modeling_xlnet.py

@@ -15,11 +15,12 @@
 from __future__ import absolute_import, division, print_function
 
 import random
+import unittest
 
 from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import CommonTestCases, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 from .utils import CACHE_DIR, require_torch, slow, torch_device
 
 
@@ -38,7 +39,7 @@ if is_torch_available():
 
 
 @require_torch
-class XLNetModelTest(CommonTestCases.CommonModelTester):
+class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
 
     all_model_classes = (
         (