Refactor some core stuff (#36539)

* some config changes

* update

* current state

* update

* update

* updates and cleanup

* something that works

* fixup

* fixes

* nits

* nit

* nits and fix

* Update src/transformers/integrations/tensor_parallel.py

Co-authored-by: Lysandre Debut <hi@lysand.re>

* Update src/transformers/integrations/tensor_parallel.py

Co-authored-by: Lysandre Debut <hi@lysand.re>

* cleanup

* style

* safe import

* fix

* updates

* rename stuff an clean

* style

* small updates

* ups

* oups

* nit

* protect imports

* update tp

* rodfl

* arf

* turbo nit on init

* fix import error

* frumble gumbgle

* try to fix the import error

* should fix the non model test

* update keep in float32

* update

* fix

* nits

* fix subvconfigs

* test was weird

* nit

* fix failing test

* fix instruct blip

* fixes

* style

* x.com

* fix overwrite

* ok last bit of failing test

---------

Co-authored-by: Lysandre Debut <hi@lysand.re>

tests/models/llava/test_configuration_llava.py

@@ -58,13 +58,13 @@ class LlavaConfigTest(unittest.TestCase):
         """
         Simple test for reloading arbirarily composed subconfigs
         """
-        default_values = LlavaConfig().to_dict()
-        default_values["vision_config"]["model_type"] = "qwen2_vl"
+        default_values = LlavaConfig().to_diff_dict()
+        default_values["vision_config"]["model_type"] = "pixtral"
         default_values["text_config"]["model_type"] = "opt"
-
+        self.maxDiff = None
         with tempfile.TemporaryDirectory() as tmp_dir:
             config = LlavaConfig(**default_values)
             config.save_pretrained(tmp_dir)
 
             reloaded = LlavaConfig.from_pretrained(tmp_dir)
-            assert config.to_dict() == reloaded.to_dict()
+            self.assertDictEqual(config.to_dict(), reloaded.to_dict())