[WIP] Test TF Flaubert + Add {XLM, Flaubert}{TokenClassification, MultipleC… (#5614)

* Test TF Flaubert + Add {XLM, Flaubert}{TokenClassification, MultipleChoice} models and tests

* AutoModels


Tiny tweaks

* Style

* Final changes before merge

* Re-order for simpler review

* Final fixes

* Addressing @sgugger's comments

* Test MultipleChoice

tests/test_modeling_xlm.py

@@ -33,6 +33,7 @@ if is_torch_available():
         XLMForQuestionAnswering,
         XLMForSequenceClassification,
         XLMForQuestionAnsweringSimple,
+        XLMForMultipleChoice,
     )
     from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST
 
@@ -63,7 +64,7 @@ class XLMModelTester:
         self.max_position_embeddings = 512
         self.type_sequence_label_size = 2
         self.initializer_range = 0.02
-        self.num_labels = 3
+        self.num_labels = 2
         self.num_choices = 4
         self.summary_type = "last"
         self.use_proj = True
@@ -91,6 +92,7 @@ class XLMModelTester:
             sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
             token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
             is_impossible_labels = ids_tensor([self.batch_size], 2).float()
+            choice_labels = ids_tensor([self.batch_size], self.num_choices)
 
         config = XLMConfig(
             vocab_size=self.vocab_size,
@@ -109,6 +111,7 @@ class XLMModelTester:
             initializer_range=self.initializer_range,
             summary_type=self.summary_type,
             use_proj=self.use_proj,
+            num_labels=self.num_labels,
             bos_token_id=self.bos_token_id,
         )
 
@@ -120,6 +123,7 @@ class XLMModelTester:
             sequence_labels,
             token_labels,
             is_impossible_labels,
+            choice_labels,
             input_mask,
         )
 
@@ -135,6 +139,7 @@ class XLMModelTester:
         sequence_labels,
         token_labels,
         is_impossible_labels,
+        choice_labels,
         input_mask,
     ):
         model = XLMModel(config=config)
@@ -160,6 +165,7 @@ class XLMModelTester:
         sequence_labels,
         token_labels,
         is_impossible_labels,
+        choice_labels,
         input_mask,
     ):
         model = XLMWithLMHeadModel(config)
@@ -185,6 +191,7 @@ class XLMModelTester:
         sequence_labels,
         token_labels,
         is_impossible_labels,
+        choice_labels,
         input_mask,
     ):
         model = XLMForQuestionAnsweringSimple(config)
@@ -214,6 +221,7 @@ class XLMModelTester:
         sequence_labels,
         token_labels,
         is_impossible_labels,
+        choice_labels,
         input_mask,
     ):
         model = XLMForQuestionAnswering(config)
@@ -280,6 +288,7 @@ class XLMModelTester:
         sequence_labels,
         token_labels,
         is_impossible_labels,
+        choice_labels,
         input_mask,
     ):
         model = XLMForSequenceClassification(config)
@@ -306,6 +315,7 @@ class XLMModelTester:
         sequence_labels,
         token_labels,
         is_impossible_labels,
+        choice_labels,
         input_mask,
     ):
         config.num_labels = self.num_labels
@@ -321,6 +331,38 @@ class XLMModelTester:
         self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels])
         self.check_loss_output(result)
 
+    def create_and_check_xlm_for_multiple_choice(
+        self,
+        config,
+        input_ids,
+        token_type_ids,
+        input_lengths,
+        sequence_labels,
+        token_labels,
+        is_impossible_labels,
+        choice_labels,
+        input_mask,
+    ):
+        config.num_choices = self.num_choices
+        model = XLMForMultipleChoice(config=config)
+        model.to(torch_device)
+        model.eval()
+        multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+        multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+        multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+        loss, logits = model(
+            multiple_choice_inputs_ids,
+            attention_mask=multiple_choice_input_mask,
+            token_type_ids=multiple_choice_token_type_ids,
+            labels=choice_labels,
+        )
+        result = {
+            "loss": loss,
+            "logits": logits,
+        }
+        self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_choices])
+        self.check_loss_output(result)
+
     def prepare_config_and_inputs_for_common(self):
         config_and_inputs = self.prepare_config_and_inputs()
         (
@@ -331,6 +373,7 @@ class XLMModelTester:
             sequence_labels,
             token_labels,
             is_impossible_labels,
+            choice_labels,
             input_mask,
         ) = config_and_inputs
         inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths}
@@ -348,6 +391,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             XLMForSequenceClassification,
             XLMForQuestionAnsweringSimple,
             XLMForTokenClassification,
+            XLMForMultipleChoice,
         )
         if is_torch_available()
         else ()
@@ -387,6 +431,10 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_token_classif(*config_and_inputs)
 
+    def test_xlm_for_multiple_choice(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_xlm_for_multiple_choice(*config_and_inputs)
+
     @slow
     def test_model_from_pretrained(self):
         for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: