[docs] Add integration test example to copy pasta template (#5961)

Co-authored-by: Julien Chaumond <chaumond@gmail.com>

CONTRIBUTING.md

@@ -215,7 +215,7 @@ Follow these steps to start contributing:
      `RUN_SLOW=1 python -m pytest tests/test_my_new_model.py`. 
    - If you are adding a new tokenizer, write tests, and make sure 
      `RUN_SLOW=1 python -m pytest tests/test_tokenization_{your_model_name}.py` passes.
-   CircleCI does not run the slow tests. 
+   CircleCI does not run the slow tests, but github actions does every night!
 6. All public methods must have informative docstrings that work nicely with sphinx. See `modeling_ctrl.py` for an 
    example.
 
@@ -239,6 +239,16 @@ $ pip install -r examples/requirements.txt  # only needed the first time
 $ python -m pytest -n auto --dist=loadfile -s -v ./examples/
 ```
 
+and for the slow tests:
+
+```bash
+RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./tests/
+```
+or 
+```python
+RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./tests/
+```
+
 In fact, that's how `make test` and `make test-examples` are implemented!
 
 You can specify a smaller set of tests in order to test only the feature

examples/deebert/test_glue_deebert.py

@@ -5,6 +5,7 @@ import unittest
 from unittest.mock import patch
 
 import run_glue_deebert
+from transformers.testing_utils import slow
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -20,6 +21,7 @@ def get_setup_file():
 
 
 class DeeBertTests(unittest.TestCase):
+    @slow
     def test_glue_deebert(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

src/transformers/testing_utils.py

@@ -128,3 +128,11 @@ if _torch_available:
     torch_device = "cuda" if parse_flag_from_env("USE_CUDA") else "cpu"
 else:
     torch_device = None
+
+
+def require_torch_and_cuda(test_case):
+    """Decorator marking a test that requires CUDA and PyTorch). """
+    if torch_device != "cuda":
+        return unittest.skip("test requires CUDA")
+    else:
+        return test_case

templates/adding_a_new_example_script/README.md

@@ -1,5 +1,15 @@
 # How to add a new example script in 🤗Transformers
 
 This folder provide a template for adding a new example script implementing a training or inference task with the models in the  🤗Transformers library.
+Add tests!
 
-Currently only examples for PyTorch are provided which are adaptations of the library's SQuAD examples which implement single-GPU and distributed training with gradient accumulation and mixed-precision (using NVIDIA's apex library) to cover a reasonable range of use cases.
+
+These folder can be put in a subdirectory under your example's name, like `examples/deebert`.
+
+
+Best Practices: 
+- use `Trainer`/`TFTrainer`
+- write an @slow test that checks that your model can train on one batch and get a low loss.
+    - this test should use cuda if it's available. (e.g. by checking `transformers.torch_device`)
+- adding an `eval_xxx.py` script that can evaluate a pretrained checkpoint.  
+- tweet about your new example with a carbon screenshot of how to run it and tag @huggingface

templates/adding_a_new_model/README.md

@@ -18,6 +18,7 @@ Here an overview of the general workflow:
 - [ ] add model/configuration/tokenization classes
 - [ ] add conversion scripts
 - [ ] add tests
+- [ ] add @slow integration test
 - [ ] finalize
 
 Let's detail what should be done at each step

templates/adding_a_new_model/modeling_xxx.py

@@ -347,23 +347,7 @@ class XxxModel(XxxPreTrainedModel):
         if token_type_ids is None:
             token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
 
-        # We create a 3D attention mask from a 2D tensor mask.
-        # (this can be done with self.invert_attention_mask)
-        # Sizes are [batch_size, 1, 1, to_seq_length]
-        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
-        # this attention mask is more simple than the triangular masking of causal attention
-        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
-
-        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
-
-        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
-        # masked positions, this operation will create a tensor which is 0.0 for
-        # positions we want to attend and -10000.0 for masked positions.
-        # Since we are adding it to the raw scores before the softmax, this is
-        # effectively the same as removing these entirely.
-        extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
-        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
-
+        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, device)
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
         # attention_probs has shape bsz x n_heads x N x N

templates/adding_a_new_model/tests/test_modeling_xxx.py

@@ -20,7 +20,7 @@ from transformers import is_torch_available
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_common import ModelTesterMixin, ids_tensor
-from .utils import CACHE_DIR, require_torch, slow, torch_device
+from .utils import require_torch, require_torch_and_cuda, slow, torch_device
 
 
 if is_torch_available():
@@ -31,20 +31,17 @@ if is_torch_available():
         XxxForQuestionAnswering,
         XxxForSequenceClassification,
         XxxForTokenClassification,
+        AutoModelForMaskedLM,
+        AutoTokenizer,
     )
-    from transformers.modeling_xxx import XXX_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.file_utils import cached_property
+
+    #
 
 
-@require_torch
-class XxxModelTest(ModelTesterMixin, unittest.TestCase):
+class XxxModelTester:
+    """You can also import this e.g from .test_modeling_bart import BartModelTester """
 
-    all_model_classes = (
-        (XxxModel, XxxForMaskedLM, XxxForQuestionAnswering, XxxForSequenceClassification, XxxForTokenClassification)
-        if is_torch_available()
-        else ()
-    )
-
-    class XxxModelTester(object):
     def __init__(
         self,
         parent,
@@ -214,16 +211,12 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
         model = XxxForTokenClassification(config=config)
         model.to(torch_device)
         model.eval()
-            loss, logits = model(
-                input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
-            )
+        loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
         result = {
             "loss": loss,
             "logits": logits,
         }
-            self.parent.assertListEqual(
-                list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
-            )
+        self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels])
         self.check_loss_output(result)
 
     def prepare_config_and_inputs_for_common(self):
@@ -240,8 +233,18 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
         inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
         return config, inputs_dict
 
+
+@require_torch
+class XxxModelTest(ModelTesterMixin, unittest.TestCase):
+
+    all_model_classes = (
+        (XxxModel, XxxForMaskedLM, XxxForQuestionAnswering, XxxForSequenceClassification, XxxForTokenClassification)
+        if is_torch_available()
+        else ()
+    )
+
     def setUp(self):
-        self.model_tester = XxxModelTest.XxxModelTester(self)
+        self.model_tester = XxxModelTester(self)
         self.config_tester = ConfigTester(self, config_class=XxxConfig, hidden_size=37)
 
     def test_config(self):
@@ -268,7 +271,50 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
         self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs)
 
     @slow
-    def test_model_from_pretrained(self):
-        for model_name in XXX_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
-            model = XxxModel.from_pretrained(model_name, cache_dir=CACHE_DIR)
-            self.assertIsNotNone(model)
+    def test_lm_outputs_same_as_reference_model(self):
+        """Write something that could help someone fixing this here."""
+        checkpoint_path = "XXX/bart-large"
+        model = self.big_model
+        tokenizer = AutoTokenizer.from_pretrained(
+            checkpoint_path
+        )  # same with AutoTokenizer (see tokenization_auto.py). This is not mandatory
+        # MODIFY THIS DEPENDING ON YOUR MODELS RELEVANT TASK.
+        batch = tokenizer(["I went to the <mask> yesterday"]).to(torch_device)
+        desired_mask_result = tokenizer.decode("store")  # update this
+        logits = model(**batch).logits
+        masked_index = (batch.input_ids == self.tokenizer.mask_token_id).nonzero()
+        assert model.num_parameters() == 175e9  # a joke
+        mask_entry_logits = logits[0, masked_index.item(), :]
+        probs = mask_entry_logits.softmax(dim=0)
+        _, predictions = probs.topk(1)
+        self.assertEqual(tokenizer.decode(predictions), desired_mask_result)
+
+    @cached_property
+    def big_model(self):
+        """Cached property means this code will only be executed once."""
+        checkpoint_path = "XXX/bart-large"
+        model = AutoModelForMaskedLM.from_pretrained(checkpoint_path).to(
+            torch_device
+        )  # test whether AutoModel can determine your model_class from checkpoint name
+        if torch_device == "cuda":
+            model.half()
+
+    # optional: do more testing! This will save you time later!
+    @slow
+    def test_that_XXX_can_be_used_in_a_pipeline(self):
+        """We can use self.big_model here without calling __init__ again."""
+        pass
+
+    def test_XXX_loss_doesnt_change_if_you_add_padding(self):
+        pass
+
+    def test_XXX_bad_args(self):
+        pass
+
+    def test_XXX_backward_pass_reduces_loss(self):
+        """Test loss/gradients same as reference implementation, for example."""
+        pass
+
+    @require_torch_and_cuda
+    def test_large_inputs_in_fp16_dont_cause_overflow(self):
+        pass

templates/adding_a_new_model/tests/test_tokenization_xxx.py

@@ -62,3 +62,7 @@ class XxxTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokens = tokenizer.tokenize("UNwant\u00E9d,running")
         self.assertListEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"])
         self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9])
+
+    def test_special_tokens_as_you_expect(self):
+        """If you are training a seq2seq model that expects a decoder_prefix token make sure it is prepended to decoder_input_ids """
+        pass