Better typing for model.config (#39132)

* Apply to all models config annotation

* Update modular to preserve order

* Apply modular

* fix define docstring

* fix dinov2 consistency (docs<->modular)

* fix InstructBlipVideoForConditionalGeneration docs<->modular consistency

* fixup

* remove duplicate code

* Delete config_class attribute from the modeling code

* Add config_class attribute in base model

* Update init sub class

* Deprecated models update

* Update new models

* Fix remote code BC issue

* fixup

* fixing more corner cases

* fix new models

* add test

* modular docs update

* fix comment a bit

* fix for py3.9

tests/utils/test_modeling_utils.py

@@ -2004,6 +2004,37 @@ class ModelUtilsTest(TestCasePlus):
             self.assertTrue(explicit_transformers_weights not in os.listdir(tmpdirname))
             self.assertTrue("model.safetensors.index.json" in os.listdir(tmpdirname))
 
+    def test_config_class_attribute(self):
+        # custom configs
+        class MyConfigA(PretrainedConfig):
+            pass
+
+        class MyConfigB(PretrainedConfig):
+            pass
+
+        class MyConfigC(PretrainedConfig):
+            pass
+
+        # custom models
+        class MyModelA(PreTrainedModel):
+            config: dict
+            config_class = MyConfigA
+
+        class MyModelB(MyModelA):
+            config: MyConfigB
+
+        class MyModelC(MyModelA):
+            config_class = MyConfigC
+
+        class MyModelD(MyModelA):
+            pass
+
+        # child config_class > child 'config:' > parent config_class > parent 'config:'
+        self.assertIs(MyModelA.config_class, MyConfigA)
+        self.assertIs(MyModelB.config_class, MyConfigB)
+        self.assertIs(MyModelC.config_class, MyConfigC)
+        self.assertIs(MyModelD.config_class, MyConfigA)
+
 
 @slow
 @require_torch