Deprecate low use models (#30781)

* Deprecate models
- graphormer
- time_series_transformer
- xlm_prophetnet
- qdqbert
- nat
- ernie_m
- tvlt
- nezha
- mega
- jukebox
- vit_hybrid
- x_clip
- deta
- speech_to_text_2
- efficientformer
- realm
- gptsan_japanese

* Fix up

* Fix speech2text2 imports

* Make sure message isn't indented

* Fix docstrings

* Correctly map for deprecated models from model_type

* Uncomment out

* Add back time series transformer and x-clip

* Import fix and fix-up

* Fix up with updated ruff

utils/check_config_attributes.py

@@ -61,8 +61,6 @@ SPECIAL_CASES_TO_ALLOW = {
     # `ignore_value` used during training (despite we don't have training script for these models yet)
     # `norm` used in conversion script (despite not using in the modeling file)
     "OneFormerConfig": ["ignore_value", "norm"],
-    # used during preprocessing and collation, see `collating_graphormer.py`
-    "GraphormerConfig": ["spatial_pos_max"],
     # used internally in the configuration class file
     "T5Config": ["feed_forward_proj"],
     # used internally in the configuration class file
@@ -134,20 +132,16 @@ SPECIAL_CASES_TO_ALLOW.update(
     {
         "CLIPSegConfig": True,
         "DeformableDetrConfig": True,
-        "DetaConfig": True,
         "DinatConfig": True,
         "DonutSwinConfig": True,
-        "EfficientFormerConfig": True,
         "FastSpeech2ConformerConfig": True,
         "FSMTConfig": True,
-        "JukeboxConfig": True,
         "LayoutLMv2Config": True,
         "MaskFormerSwinConfig": True,
         "MT5Config": True,
         # For backward compatibility with trust remote code models
         "MptConfig": True,
         "MptAttentionConfig": True,
-        "NatConfig": True,
         "OneFormerConfig": True,
         "PerceiverConfig": True,
         "RagConfig": True,