Add strong test for configuration attributes (#14000)

* Add strong test for configuration attributes

* Add fake modif to trigger all tests

* Add a better fake modif

* Ignore is_encoder_decoder

* Fix faulty configs

* Remove fake modif

tests/test_configuration_common.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import copy
 import json
 import os
 import tempfile
@@ -21,10 +21,62 @@ import unittest
 
 from huggingface_hub import HfApi
 from requests.exceptions import HTTPError
-from transformers import BertConfig, GPT2Config
+from transformers import BertConfig, GPT2Config, is_torch_available
+from transformers.configuration_utils import PretrainedConfig
 from transformers.testing_utils import ENDPOINT_STAGING, PASS, USER, is_staging_test
 
 
+config_common_kwargs = {
+    "return_dict": False,
+    "output_hidden_states": True,
+    "output_attentions": True,
+    "torchscript": True,
+    "torch_dtype": "float16",
+    "use_bfloat16": True,
+    "pruned_heads": {"a": 1},
+    "tie_word_embeddings": False,
+    "is_decoder": True,
+    "cross_attention_hidden_size": 128,
+    "add_cross_attention": True,
+    "tie_encoder_decoder": True,
+    "max_length": 50,
+    "min_length": 3,
+    "do_sample": True,
+    "early_stopping": True,
+    "num_beams": 3,
+    "num_beam_groups": 3,
+    "diversity_penalty": 0.5,
+    "temperature": 2.0,
+    "top_k": 10,
+    "top_p": 0.7,
+    "repetition_penalty": 0.8,
+    "length_penalty": 0.8,
+    "no_repeat_ngram_size": 5,
+    "encoder_no_repeat_ngram_size": 5,
+    "bad_words_ids": [1, 2, 3],
+    "num_return_sequences": 3,
+    "chunk_size_feed_forward": 5,
+    "output_scores": True,
+    "return_dict_in_generate": True,
+    "forced_bos_token_id": 2,
+    "forced_eos_token_id": 3,
+    "remove_invalid_values": True,
+    "architectures": ["BertModel"],
+    "finetuning_task": "translation",
+    "id2label": {0: "label"},
+    "label2id": {"label": "0"},
+    "tokenizer_class": "BertTokenizerFast",
+    "prefix": "prefix",
+    "bos_token_id": 6,
+    "pad_token_id": 7,
+    "eos_token_id": 8,
+    "sep_token_id": 9,
+    "decoder_start_token_id": 10,
+    "task_specific_params": {"translation": "some_params"},
+    "problem_type": "regression",
+}
+
+
 class ConfigTester(object):
     def __init__(self, parent, config_class=None, has_text_modality=True, **kwargs):
         self.parent = parent
@@ -108,6 +160,26 @@ class ConfigTester(object):
         config = self.config_class()
         self.parent.assertIsNotNone(config)
 
+    def check_config_arguments_init(self):
+        kwargs = copy.deepcopy(config_common_kwargs)
+        config = self.config_class(**kwargs)
+        wrong_values = []
+        for key, value in config_common_kwargs.items():
+            if key == "torch_dtype":
+                if not is_torch_available():
+                    continue
+                else:
+                    import torch
+
+                    if config.torch_dtype != torch.float16:
+                        wrong_values.append(("torch_dtype", config.torch_dtype, torch.float16))
+            elif getattr(config, key) != value:
+                wrong_values.append((key, getattr(config, key), value))
+
+        if len(wrong_values) > 0:
+            errors = "\n".join([f"- {v[0]}: got {v[1]} instead of {v[2]}" for v in wrong_values])
+            raise ValueError(f"The following keys were not properly sey in the config:\n{errors}")
+
     def run_common_tests(self):
         self.create_and_test_config_common_properties()
         self.create_and_test_config_to_json_string()
@@ -115,6 +187,7 @@ class ConfigTester(object):
         self.create_and_test_config_from_and_save_pretrained()
         self.create_and_test_config_with_num_labels()
         self.check_config_can_be_init_without_params()
+        self.check_config_arguments_init()
 
 
 @is_staging_test
@@ -183,3 +256,15 @@ class ConfigTestUtils(unittest.TestCase):
         self.assertEqual(resid_pdrop, c.resid_pdrop, "mismatch for key: resid_pdrop")
         self.assertEqual(scale_attn_weights, c.scale_attn_weights, "mismatch for key: scale_attn_weights")
         self.assertEqual(summary_type, c.summary_type, "mismatch for key: summary_type")
+
+    def test_config_common_kwargs_is_complete(self):
+        base_config = PretrainedConfig()
+        missing_keys = [key for key in base_config.__dict__ if key not in config_common_kwargs]
+        # If this part of the test fails, you have arguments to addin config_common_kwargs above.
+        self.assertListEqual(missing_keys, ["is_encoder_decoder", "_name_or_path", "transformers_version"])
+        keys_with_defaults = [key for key, value in config_common_kwargs.items() if value == getattr(base_config, key)]
+        if len(keys_with_defaults) > 0:
+            raise ValueError(
+                "The following keys are set with the default values in `test_configuration_common.config_common_kwargs` "
+                f"pick another value for them: {', '.join(keys_with_defaults)}."
+            )