From 5f5ccfdc545d94202ded13217613032c81151aa5 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Thu, 8 May 2025 23:46:07 +0200 Subject: [PATCH] [`AutoDocstring`] Based on inspect parsing of the signature (#33771) * delete common docstring * nit * updates * push * fixup * move stuff around fixup * no need for dataclas * damn nice modular * add auto class docstring * style * modular update * import autodocstring * fixup * maybe add original doc! * more cleanup * remove class do cas well * update * nits * more celanup * fix * wups * small check * updatez * some fixes * fix doc * update * nits * try? * nit * some updates * a little bit better * where ever we did not have help we are not really adding it! * revert llama config * small fixes and small tests * test * fixup * more fix-copies * updates * updates * fix doc building * style * small fixes * nits * fix-copies * fix merge issues faster * fix merge conf * nits jamba * ? * working autodoc for model class and forward except returns and example * support return section and unpack kwargs description * nits and cleanup * fix-copies * fix-copies * nits * Add support for llava-like models * fixup * add class args subset support * add examples inferred from automodel/pipelines * update ruff * autodocstring for Aria, Albert + fixups * Fix empty return blocks * fix copies * fix copies * add autodoc for all fast image processors + align, altclip * fix copies * add auto_doc for audio_spectrogram, auto_former, bark, bamba * Drastically improve speed + add bart beit bert * add autodoc to all bert-like models * Fix broken doc * fix copies * fix auto_docstring after merge * add autodoc to models * add models * add models * add models and improve support for optional, and custom shape in args docstring * update fast image processors * refactor auto_method_docstring in args_doc * add models and fix docstring parsing * add models * add models * remove debugging * add models * add fix_auto_docstrings and improve args_docs * add support for additional_info in args docstring * refactor (almost) all models * fix check docstring * fix -copies * fill in all missing docstrings * fix copies * fix qwen3 moe docstring * add documentation * add back labels * update docs and fix can_return_tuple in modular files * fix LongformerForMaskedLM docstring * add auto_docstring to _toctree * remove auto_docstring tests temporarily * fix copyrights new files * fix can_return_tuple granite hybrid * fix fast beit * Fix empty config doc * add support for COMMON_CUSTOM_ARGS in check_docstrings and add missing models * fix code block not closed flava * fix can_return_tuple sam hq * Fix Flaubert dataclass --------- Co-authored-by: yonigozlan Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> --- docs/source/en/_toctree.yml | 2 + docs/source/en/auto_docstring.md | 279 +++ .../commands/add_fast_image_processor.py | 10 +- .../image_processing_utils_fast.py | 106 +- .../models/albert/modeling_albert.py | 221 +-- .../models/align/modeling_align.py | 190 +- .../models/altclip/modeling_altclip.py | 182 +- src/transformers/models/aria/modeling_aria.py | 238 +-- src/transformers/models/aria/modular_aria.py | 118 +- .../modeling_audio_spectrogram_transformer.py | 103 +- .../models/autoformer/modeling_autoformer.py | 322 ++-- .../models/aya_vision/modeling_aya_vision.py | 148 +- .../models/aya_vision/modular_aya_vision.py | 37 +- .../models/bamba/modeling_bamba.py | 140 +- .../models/bamba/modular_bamba.py | 149 +- src/transformers/models/bark/modeling_bark.py | 197 +-- src/transformers/models/bart/modeling_bart.py | 480 ++--- .../models/beit/image_processing_beit_fast.py | 39 +- src/transformers/models/beit/modeling_beit.py | 127 +- src/transformers/models/bert/modeling_bert.py | 339 +--- .../modeling_bert_generation.py | 134 +- .../models/big_bird/modeling_big_bird.py | 255 +-- .../modeling_bigbird_pegasus.py | 381 ++-- .../models/biogpt/modeling_biogpt.py | 133 +- .../models/bit/image_processing_bit_fast.py | 12 +- src/transformers/models/bit/modeling_bit.py | 93 +- .../models/bitnet/modeling_bitnet.py | 132 +- .../models/bitnet/modular_bitnet.py | 17 +- .../models/blenderbot/modeling_blenderbot.py | 319 +--- .../modeling_blenderbot_small.py | 319 +--- .../models/blip/image_processing_blip_fast.py | 9 +- src/transformers/models/blip/modeling_blip.py | 182 +- .../models/blip_2/modeling_blip_2.py | 407 +---- .../models/bloom/modeling_bloom.py | 220 +-- .../image_processing_bridgetower_fast.py | 33 +- .../bridgetower/modeling_bridgetower.py | 182 +- src/transformers/models/bros/modeling_bros.py | 177 +- .../models/camembert/modeling_camembert.py | 314 +--- .../models/canine/modeling_canine.py | 187 +- .../models/chameleon/modeling_chameleon.py | 172 +- .../image_processing_chinese_clip_fast.py | 9 +- .../chinese_clip/modeling_chinese_clip.py | 212 +-- src/transformers/models/clap/modeling_clap.py | 218 +-- .../models/clip/image_processing_clip_fast.py | 9 +- src/transformers/models/clip/modeling_clip.py | 220 +-- .../models/clipseg/modeling_clipseg.py | 172 +- src/transformers/models/clvp/modeling_clvp.py | 167 +- .../models/codegen/modeling_codegen.py | 132 +- .../models/cohere/modeling_cohere.py | 139 +- .../models/cohere/modular_cohere.py | 19 +- .../models/cohere2/modeling_cohere2.py | 139 +- .../models/cohere2/modular_cohere2.py | 14 +- .../models/colpali/modeling_colpali.py | 83 +- .../image_processing_conditional_detr_fast.py | 85 +- .../modeling_conditional_detr.py | 134 +- .../models/convbert/modeling_convbert.py | 195 +- .../image_processing_convnext_fast.py | 29 +- .../models/convnext/modeling_convnext.py | 93 +- .../models/convnextv2/modeling_convnextv2.py | 92 +- .../models/cpmant/modeling_cpmant.py | 111 +- src/transformers/models/csm/modeling_csm.py | 288 +-- src/transformers/models/csm/modular_csm.py | 252 +-- src/transformers/models/ctrl/modeling_ctrl.py | 156 +- src/transformers/models/cvt/modeling_cvt.py | 77 +- .../models/d_fine/modeling_d_fine.py | 107 +- .../models/dab_detr/modeling_dab_detr.py | 102 +- src/transformers/models/dac/modeling_dac.py | 103 +- .../data2vec/modeling_data2vec_audio.py | 184 +- .../models/data2vec/modeling_data2vec_text.py | 260 +-- .../data2vec/modeling_data2vec_vision.py | 107 +- .../models/data2vec/modular_data2vec_audio.py | 159 +- src/transformers/models/dbrx/modeling_dbrx.py | 143 +- .../models/deberta/modeling_deberta.py | 165 +- .../models/deberta_v2/modeling_deberta_v2.py | 167 +- .../modeling_decision_transformer.py | 63 +- .../deepseek_v3/modeling_deepseek_v3.py | 138 +- .../image_processing_deformable_detr_fast.py | 85 +- .../modeling_deformable_detr.py | 102 +- .../models/deit/image_processing_deit_fast.py | 9 +- src/transformers/models/deit/modeling_deit.py | 123 +- .../depth_anything/modeling_depth_anything.py | 53 +- .../depth_pro/image_processing_depth_pro.py | 11 +- .../image_processing_depth_pro_fast.py | 8 +- .../models/depth_pro/modeling_depth_pro.py | 90 +- .../models/detr/image_processing_detr_fast.py | 85 +- src/transformers/models/detr/modeling_detr.py | 122 +- .../models/diffllama/modeling_diffllama.py | 186 +- .../models/dinat/modeling_dinat.py | 97 +- .../models/dinov2/modeling_dinov2.py | 141 +- .../modeling_dinov2_with_registers.py | 141 +- .../modular_dinov2_with_registers.py | 2 - .../models/distilbert/modeling_distilbert.py | 205 +-- .../donut/image_processing_donut_fast.py | 37 +- .../models/donut/modeling_donut_swin.py | 96 +- src/transformers/models/dpr/modeling_dpr.py | 205 +-- src/transformers/models/dpt/modeling_dpt.py | 97 +- .../image_processing_efficientnet_fast.py | 27 +- .../efficientnet/modeling_efficientnet.py | 78 +- .../models/electra/modeling_electra.py | 265 +-- src/transformers/models/emu3/modeling_emu3.py | 277 +-- src/transformers/models/emu3/modular_emu3.py | 256 +-- .../models/encodec/modeling_encodec.py | 97 +- .../modeling_encoder_decoder.py | 140 +- .../models/ernie/modeling_ernie.py | 338 ++-- src/transformers/models/esm/modeling_esm.py | 157 +- .../models/esm/modeling_esmfold.py | 62 +- .../models/falcon/modeling_falcon.py | 240 +-- .../falcon_mamba/modeling_falcon_mamba.py | 100 +- .../modeling_fastspeech2_conformer.py | 199 +-- .../models/flaubert/modeling_flaubert.py | 403 ++--- .../flava/image_processing_flava_fast.py | 121 +- .../models/flava/modeling_flava.py | 367 ++-- src/transformers/models/fnet/modeling_fnet.py | 209 +-- .../models/focalnet/modeling_focalnet.py | 113 +- src/transformers/models/fsmt/modeling_fsmt.py | 207 +-- .../models/funnel/modeling_funnel.py | 195 +- src/transformers/models/fuyu/modeling_fuyu.py | 144 +- .../models/gemma/modeling_gemma.py | 165 +- .../models/gemma/modular_gemma.py | 15 - .../models/gemma2/modeling_gemma2.py | 166 +- .../models/gemma2/modular_gemma2.py | 23 +- .../gemma3/image_processing_gemma3_fast.py | 44 +- .../models/gemma3/modeling_gemma3.py | 196 +- .../models/gemma3/modular_gemma3.py | 43 +- src/transformers/models/git/modeling_git.py | 165 +- src/transformers/models/glm/modeling_glm.py | 165 +- src/transformers/models/glm4/modeling_glm4.py | 165 +- src/transformers/models/glm4/modular_glm4.py | 24 +- src/transformers/models/glpn/modeling_glpn.py | 77 +- .../image_processing_got_ocr2_fast.py | 51 +- .../models/got_ocr2/modeling_got_ocr2.py | 140 +- .../models/got_ocr2/modular_got_ocr2.py | 103 +- src/transformers/models/gpt2/modeling_gpt2.py | 302 +--- .../gpt_bigcode/modeling_gpt_bigcode.py | 193 +- .../models/gpt_neo/modeling_gpt_neo.py | 256 +-- .../models/gpt_neox/modeling_gpt_neox.py | 193 +- .../models/gpt_neox/modular_gpt_neox.py | 93 +- .../modeling_gpt_neox_japanese.py | 117 +- src/transformers/models/gptj/modeling_gptj.py | 180 +- .../models/granite/modeling_granite.py | 138 +- .../granite_speech/modeling_granite_speech.py | 143 +- .../models/granitemoe/modeling_granitemoe.py | 146 +- .../modeling_granitemoehybrid.py | 143 +- .../modular_granitemoehybrid.py | 122 +- .../modeling_granitemoeshared.py | 141 +- .../modular_granitemoeshared.py | 111 +- .../image_processing_grounding_dino_fast.py | 85 +- .../grounding_dino/modeling_grounding_dino.py | 121 +- .../models/groupvit/modeling_groupvit.py | 151 +- .../models/helium/modeling_helium.py | 165 +- .../models/helium/modular_helium.py | 18 +- .../models/hgnet_v2/modeling_hgnet_v2.py | 54 +- .../models/hgnet_v2/modular_hgnet_v2.py | 55 +- .../models/hiera/modeling_hiera.py | 133 +- .../models/hubert/modeling_hubert.py | 145 +- .../models/hubert/modular_hubert.py | 135 +- .../models/ibert/modeling_ibert.py | 198 +-- .../models/idefics/modeling_idefics.py | 134 +- .../models/idefics2/modeling_idefics2.py | 219 +-- .../models/idefics3/modeling_idefics3.py | 183 +- .../models/ijepa/modeling_ijepa.py | 98 +- .../models/ijepa/modular_ijepa.py | 55 +- .../models/imagegpt/modeling_imagegpt.py | 155 +- .../models/informer/modeling_informer.py | 389 ++-- .../instructblip/modeling_instructblip.py | 208 +-- .../modeling_instructblipvideo.py | 207 +-- .../modular_instructblipvideo.py | 12 +- .../models/internvl/modeling_internvl.py | 201 +-- .../models/internvl/modular_internvl.py | 90 +- .../models/jamba/modeling_jamba.py | 154 +- .../models/janus/modeling_janus.py | 192 +- .../models/janus/modular_janus.py | 163 +- .../models/jetmoe/modeling_jetmoe.py | 114 +- .../models/kosmos2/modeling_kosmos2.py | 280 +-- .../models/layoutlm/modeling_layoutlm.py | 163 +- .../image_processing_layoutlmv2_fast.py | 45 +- .../models/layoutlmv2/modeling_layoutlmv2.py | 234 ++- .../image_processing_layoutlmv3_fast.py | 39 +- .../models/layoutlmv3/modeling_layoutlmv3.py | 281 +-- src/transformers/models/led/modeling_led.py | 411 ++--- .../levit/image_processing_levit_fast.py | 9 +- .../models/levit/modeling_levit.py | 88 +- src/transformers/models/lilt/modeling_lilt.py | 162 +- .../models/llama/configuration_llama.py | 1 + .../models/llama/modeling_llama.py | 186 +- .../llama4/image_processing_llama4_fast.py | 52 +- .../models/llama4/modeling_llama4.py | 181 +- .../llava/image_processing_llava_fast.py | 27 +- .../models/llava/modeling_llava.py | 151 +- .../image_processing_llava_next_fast.py | 40 +- .../models/llava_next/modeling_llava_next.py | 160 +- .../modeling_llava_next_video.py | 164 +- .../modular_llava_next_video.py | 124 +- .../image_processing_llava_onevision_fast.py | 40 +- .../modeling_llava_onevision.py | 168 +- .../modular_llava_onevision.py | 74 +- .../models/longformer/modeling_longformer.py | 286 ++- .../models/longt5/modeling_longt5.py | 288 ++- src/transformers/models/luke/modeling_luke.py | 436 ++--- .../models/lxmert/modeling_lxmert.py | 190 +- .../models/m2m_100/modeling_m2m_100.py | 217 +-- .../models/mamba/modeling_mamba.py | 100 +- .../models/mamba2/modeling_mamba2.py | 112 +- .../models/marian/modeling_marian.py | 296 +--- .../models/markuplm/modeling_markuplm.py | 149 +- .../mask2former/modeling_mask2former.py | 102 +- .../models/maskformer/modeling_maskformer.py | 59 +- .../models/mbart/modeling_mbart.py | 449 ++--- .../megatron_bert/modeling_megatron_bert.py | 284 +-- .../models/mgp_str/modeling_mgp_str.py | 67 +- src/transformers/models/mimi/modeling_mimi.py | 104 +- .../models/mistral/modeling_mistral.py | 175 +- .../models/mistral3/modeling_mistral3.py | 142 +- .../models/mistral3/modular_mistral3.py | 33 +- .../models/mixtral/modeling_mixtral.py | 176 +- .../models/mixtral/modular_mixtral.py | 30 +- src/transformers/models/mlcd/modeling_mlcd.py | 68 +- src/transformers/models/mlcd/modular_mlcd.py | 72 +- .../models/mllama/modeling_mllama.py | 442 ++--- .../models/mobilebert/modeling_mobilebert.py | 255 +-- .../image_processing_mobilenet_v1_fast.py | 8 +- .../mobilenet_v1/modeling_mobilenet_v1.py | 77 +- .../image_processing_mobilenet_v2_fast.py | 9 +- .../mobilenet_v2/modeling_mobilenet_v2.py | 95 +- .../models/mobilevit/modeling_mobilevit.py | 97 +- .../mobilevitv2/modeling_mobilevitv2.py | 96 +- .../models/modernbert/modeling_modernbert.py | 240 +-- .../models/modernbert/modular_modernbert.py | 238 +-- .../models/moonshine/modeling_moonshine.py | 335 ++-- .../models/moonshine/modular_moonshine.py | 204 +-- .../models/moshi/modeling_moshi.py | 264 +-- .../models/mpnet/modeling_mpnet.py | 173 +- src/transformers/models/mpt/modeling_mpt.py | 202 +-- src/transformers/models/mra/modeling_mra.py | 188 +- src/transformers/models/mt5/modeling_mt5.py | 409 ++--- .../models/musicgen/modeling_musicgen.py | 397 ++--- .../modeling_musicgen_melody.py | 359 ++-- src/transformers/models/mvp/modeling_mvp.py | 549 +++--- .../models/nemotron/modeling_nemotron.py | 183 +- .../models/nllb_moe/modeling_nllb_moe.py | 216 +-- .../nystromformer/modeling_nystromformer.py | 191 +- src/transformers/models/olmo/modeling_olmo.py | 138 +- .../models/olmo2/modeling_olmo2.py | 138 +- .../models/olmoe/modeling_olmoe.py | 149 +- .../omdet_turbo/modeling_omdet_turbo.py | 107 +- .../models/oneformer/modeling_oneformer.py | 77 +- .../models/openai/modeling_openai.py | 138 +- src/transformers/models/opt/modeling_opt.py | 240 +-- .../models/owlv2/modeling_owlv2.py | 203 +-- .../owlvit/image_processing_owlvit_fast.py | 12 +- .../models/owlvit/modeling_owlvit.py | 201 +-- .../models/paligemma/modeling_paligemma.py | 169 +- .../patchtsmixer/modeling_patchtsmixer.py | 172 +- .../models/patchtst/modeling_patchtst.py | 131 +- .../models/pegasus/modeling_pegasus.py | 296 +--- .../models/pegasus_x/modeling_pegasus_x.py | 176 +- .../image_processing_perceiver_fast.py | 9 +- .../models/perceiver/modeling_perceiver.py | 328 ++-- .../models/persimmon/modeling_persimmon.py | 161 +- src/transformers/models/phi/modeling_phi.py | 165 +- src/transformers/models/phi3/modeling_phi3.py | 165 +- .../modeling_phi4_multimodal.py | 167 +- .../modular_phi4_multimodal.py | 130 +- .../models/phimoe/modeling_phimoe.py | 144 +- .../models/pix2struct/modeling_pix2struct.py | 346 +--- .../pixtral/image_processing_pixtral_fast.py | 26 +- .../models/pixtral/modeling_pixtral.py | 55 +- .../models/plbart/modeling_plbart.py | 368 ++-- .../image_processing_poolformer_fast.py | 27 +- .../models/poolformer/modeling_poolformer.py | 67 +- .../models/pop2piano/modeling_pop2piano.py | 157 +- .../modeling_prompt_depth_anything.py | 69 +- .../modular_prompt_depth_anything.py | 69 +- .../models/prophetnet/modeling_prophetnet.py | 295 +--- .../models/pvt/image_processing_pvt_fast.py | 12 +- src/transformers/models/pvt/modeling_pvt.py | 77 +- .../models/pvt_v2/modeling_pvt_v2.py | 92 +- .../models/qwen2/modeling_qwen2.py | 186 +- .../qwen2_5_omni/modeling_qwen2_5_omni.py | 287 +-- .../qwen2_5_omni/modular_qwen2_5_omni.py | 261 +-- .../models/qwen2_5_vl/modeling_qwen2_5_vl.py | 158 +- .../models/qwen2_5_vl/modular_qwen2_5_vl.py | 52 +- .../qwen2_audio/modeling_qwen2_audio.py | 156 +- .../models/qwen2_moe/modeling_qwen2_moe.py | 181 +- .../image_processing_qwen2_vl_fast.py | 103 +- .../models/qwen2_vl/modeling_qwen2_vl.py | 161 +- .../models/qwen3/modeling_qwen3.py | 186 +- .../models/qwen3/modular_qwen3.py | 22 +- .../qwen3_moe/configuration_qwen3_moe.py | 2 +- .../models/qwen3_moe/modeling_qwen3_moe.py | 187 +- .../models/qwen3_moe/modular_qwen3_moe.py | 30 +- src/transformers/models/rag/modeling_rag.py | 281 +-- .../modeling_recurrent_gemma.py | 103 +- .../models/reformer/modeling_reformer.py | 264 ++- .../models/regnet/modeling_regnet.py | 74 +- .../models/rembert/modeling_rembert.py | 250 +-- .../models/resnet/modeling_resnet.py | 93 +- .../models/roberta/modeling_roberta.py | 318 +--- .../modeling_roberta_prelayernorm.py | 314 ++-- .../models/roc_bert/modeling_roc_bert.py | 467 +++-- .../models/roformer/modeling_roformer.py | 237 +-- .../rt_detr/image_processing_rt_detr_fast.py | 85 +- .../models/rt_detr/modeling_rt_detr.py | 101 +- .../models/rt_detr/modeling_rt_detr_resnet.py | 64 +- .../models/rt_detr/modular_rt_detr.py | 48 +- .../models/rt_detr_v2/modeling_rt_detr_v2.py | 107 +- src/transformers/models/rwkv/modeling_rwkv.py | 134 +- src/transformers/models/sam/modeling_sam.py | 187 +- .../models/sam_hq/modeling_sam_hq.py | 257 +-- .../models/sam_hq/modular_sam_hq.py | 151 +- .../seamless_m4t/modeling_seamless_m4t.py | 397 ++--- .../modeling_seamless_m4t_v2.py | 465 ++--- .../models/segformer/modeling_segformer.py | 96 +- .../models/seggpt/modeling_seggpt.py | 122 +- src/transformers/models/sew/modeling_sew.py | 140 +- .../models/sew_d/modeling_sew_d.py | 134 +- .../shieldgemma2/modeling_shieldgemma2.py | 85 +- .../siglip/image_processing_siglip_fast.py | 9 +- .../models/siglip/modeling_siglip.py | 186 +- .../siglip2/image_processing_siglip2_fast.py | 35 +- .../models/siglip2/modeling_siglip2.py | 211 +-- .../models/siglip2/modular_siglip2.py | 130 +- .../models/smolvlm/modeling_smolvlm.py | 186 +- .../models/smolvlm/modular_smolvlm.py | 19 +- .../modeling_speech_encoder_decoder.py | 172 +- .../speech_to_text/modeling_speech_to_text.py | 188 +- .../models/speecht5/modeling_speecht5.py | 282 +-- .../models/splinter/modeling_splinter.py | 186 +- .../squeezebert/modeling_squeezebert.py | 225 +-- .../models/stablelm/modeling_stablelm.py | 170 +- .../models/starcoder2/modeling_starcoder2.py | 165 +- .../models/starcoder2/modular_starcoder2.py | 14 +- .../models/superglue/modeling_superglue.py | 48 +- .../models/superpoint/modeling_superpoint.py | 46 +- .../swiftformer/modeling_swiftformer.py | 85 +- src/transformers/models/swin/modeling_swin.py | 123 +- .../swin2sr/image_processing_swin2sr_fast.py | 37 +- .../models/swin2sr/modeling_swin2sr.py | 81 +- .../models/swinv2/modeling_swinv2.py | 134 +- .../modeling_switch_transformers.py | 298 ++-- src/transformers/models/t5/modeling_t5.py | 407 ++--- .../modeling_table_transformer.py | 101 +- .../models/tapas/modeling_tapas.py | 170 +- .../models/textnet/modeling_textnet.py | 87 +- .../modeling_time_series_transformer.py | 388 ++-- .../models/timesfm/modeling_timesfm.py | 102 +- .../models/timesfm/modular_timesfm.py | 104 +- .../timesformer/modeling_timesformer.py | 66 +- .../timm_wrapper/modeling_timm_wrapper.py | 48 +- .../models/trocr/modeling_trocr.py | 118 +- src/transformers/models/tvp/modeling_tvp.py | 80 +- src/transformers/models/udop/modeling_udop.py | 288 +-- src/transformers/models/umt5/modeling_umt5.py | 414 ++--- .../models/unispeech/modeling_unispeech.py | 165 +- .../models/unispeech/modular_unispeech.py | 158 +- .../unispeech_sat/modeling_unispeech_sat.py | 211 +-- .../unispeech_sat/modular_unispeech_sat.py | 213 +-- .../models/univnet/modeling_univnet.py | 82 +- .../models/upernet/modeling_upernet.py | 50 +- .../video_llava/modeling_video_llava.py | 168 +- .../models/videomae/modeling_videomae.py | 83 +- src/transformers/models/vilt/modeling_vilt.py | 233 +-- .../models/vipllava/modeling_vipllava.py | 155 +- .../models/vipllava/modular_vipllava.py | 44 +- .../modeling_vision_encoder_decoder.py | 130 +- .../modeling_vision_text_dual_encoder.py | 135 +- .../visual_bert/modeling_visual_bert.py | 305 ++-- .../models/vit/image_processing_vit_fast.py | 8 +- src/transformers/models/vit/modeling_vit.py | 110 +- .../models/vit_mae/modeling_vit_mae.py | 87 +- .../models/vit_msn/modeling_vit_msn.py | 80 +- .../models/vitdet/modeling_vitdet.py | 76 +- .../image_processing_vitmatte_fast.py | 39 +- .../models/vitmatte/modeling_vitmatte.py | 57 +- .../models/vitpose/modeling_vitpose.py | 66 +- .../modeling_vitpose_backbone.py | 71 +- src/transformers/models/vits/modeling_vits.py | 79 +- .../models/vivit/modeling_vivit.py | 97 +- .../models/wav2vec2/modeling_wav2vec2.py | 214 +-- .../wav2vec2_bert/modeling_wav2vec2_bert.py | 186 +- .../wav2vec2_bert/modular_wav2vec2_bert.py | 151 +- .../modeling_wav2vec2_conformer.py | 199 +-- .../modular_wav2vec2_conformer.py | 165 +- .../models/wavlm/modeling_wavlm.py | 190 +- .../models/wavlm/modular_wavlm.py | 188 +- .../models/whisper/modeling_whisper.py | 330 ++-- .../models/x_clip/modeling_x_clip.py | 148 +- src/transformers/models/xglm/modeling_xglm.py | 165 +- src/transformers/models/xlm/modeling_xlm.py | 411 ++--- .../xlm_roberta/modeling_xlm_roberta.py | 326 +--- .../xlm_roberta_xl/modeling_xlm_roberta_xl.py | 258 +-- .../models/xlnet/modeling_xlnet.py | 467 ++--- src/transformers/models/xmod/modeling_xmod.py | 246 +-- .../yolos/image_processing_yolos_fast.py | 85 +- .../models/yolos/modeling_yolos.py | 86 +- src/transformers/models/yoso/modeling_yoso.py | 184 +- .../models/zamba/modeling_zamba.py | 151 +- .../models/zamba2/modeling_zamba2.py | 132 +- .../models/zoedepth/modeling_zoedepth.py | 56 +- src/transformers/utils/__init__.py | 11 + src/transformers/utils/args_doc.py | 1572 +++++++++++++++++ src/transformers/utils/doc.py | 402 ++++- tests/utils/test_auto_docstring.py | 88 + utils/check_docstrings.py | 406 ++++- utils/modular_model_converter.py | 26 +- 405 files changed, 18189 insertions(+), 46715 deletions(-) create mode 100644 docs/source/en/auto_docstring.md create mode 100644 src/transformers/utils/args_doc.py create mode 100644 tests/utils/test_auto_docstring.py diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index b848976d02..0dc62cc6f9 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -21,6 +21,8 @@ title: Adding a new model to Transformers - local: modular_transformers title: Modular Transformers + - local: auto_docstring + title: Document your models - local: task_summary title: What 🤗 Transformers can do - local: tasks_explained diff --git a/docs/source/en/auto_docstring.md b/docs/source/en/auto_docstring.md new file mode 100644 index 0000000000..19058c00eb --- /dev/null +++ b/docs/source/en/auto_docstring.md @@ -0,0 +1,279 @@ + + +# Utilizing the @auto_docstring Decorator + +The `@auto_docstring` decorator in the Hugging Face Transformers library helps generate docstrings for model classes and their methods, which will be used to build the documentation for the library. It aims to improve consistency and reduce boilerplate by automatically including standard argument descriptions and allowing for targeted overrides and additions. + +--- + +## 📜 How it Works + +The `@auto_docstring` decorator constructs docstrings by: + +1. **Signature Inspection:** It inspects the signature (arguments, types, defaults) of the decorated class's `__init__` method or the decorated function. +2. **Centralized Docstring Fetching:** It retrieves predefined docstrings for common arguments (e.g., `input_ids`, `attention_mask`) from internal library sources (like `ModelArgs` or `ImageProcessorArgs` in `utils/args_doc.py`). +3. **Overriding or Adding Arguments Descriptions:** + * **Direct Docstring Block:** It incorporates custom docstring content from an `r""" """` (or `""" """`) block below the method signature or within the `__init__` docstring. This is for documenting new arguments or overriding standard descriptions. + * **Decorator Arguments (`custom_args`):** A `custom_args` docstring block can be passed to the decorator to provide docstrings for specific arguments directly in the decorator call. This can be used to define the docstring block for new arguments once if they are repeated in multiple places in the modeling file. +4. **Adding Classes and Functions Introduction:** + * **`custom_intro` argument:** Allows prepending a custom introductory paragraph to a class or function docstring. + * **Automatic Introduction Generation:** For model classes with standard naming patterns (like `ModelForCausalLM`) or belonging to a pipeline, the decorator automatically generates an appropriate introductory paragraph using `ClassDocstring` in `utils/args_doc.py` as the source. +5. **Templating:** The decorator uses a templating system, allowing predefined docstrings to include dynamic information deduced from the `auto_modules` of the library, such as `{{processor_class}}` or `{{config_class}}`. +6. **Deducing Relevant Examples:** The decorator attempts to find appropriate usage examples based on the model's task or pipeline compatibility. It extracts checkpoint information from the model's configuration class to provide concrete examples with real model identifiers. +7. **Adding Return Value Documentation:** For methods like `forward`, the decorator can automatically generate the "Returns" section based on the method's return type annotation. For example, for a method returning a `ModelOutput` subclass, it will extracts field descriptions from that class's docstring to create a comprehensive return value description. A custom `Returns` section can also be manually specified in the function docstring block. +8. **Unrolling Kwargs Typed With Unpack Operator:** For specific methods (defined in `UNROLL_KWARGS_METHODS`) or classes (defined in `UNROLL_KWARGS_CLASSES`), the decorator processes `**kwargs` parameters that are typed with `Unpack[KwargsTypedDict]`. It extracts the documentation from the TypedDict and adds each parameter to the function's docstring. Currently, this functionality is only supported for `FastImageProcessorKwargs`. + + +--- + +## 🚀 How to Use @auto_docstring + +### 1. Importing the Decorator +Import the decorator into your modeling file: + +```python +from ...utils import auto_docstring +``` + +### 2. Applying to Classes +Place `@auto_docstring` directly above the class definition. It uses the `__init__` method's signature and its docstring for parameter descriptions. + +```python +from transformers.modeling_utils import PreTrainedModel +from ...utils import auto_docstring + +@auto_docstring +class MyAwesomeModel(PreTrainedModel): + def __init__(self, config, custom_parameter: int = 10, another_custom_arg: str = "default"): + r""" + custom_parameter (`int`, *optional*, defaults to 10): + Description of the custom_parameter for MyAwesomeModel. + another_custom_arg (`str`, *optional*, defaults to "default"): + Documentation for another unique argument. + """ + super().__init__(config) + self.custom_parameter = custom_parameter + self.another_custom_arg = another_custom_arg + # ... rest of your init + + # ... other methods +``` + +#### Advanced Class Decoration: + +Arguments can be passed directly to `@auto_docstring` for more control: + +```python +@auto_docstring( + custom_intro="""This model performs specific synergistic operations. + It builds upon the standard Transformer architecture with unique modifications.""", + custom_args=""" + custom_parameter (`type`, *optional*, defaults to `default_value`): + A concise description for custom_parameter if not defined or overriding the description in `args_doc.py`. + internal_helper_arg (`type`, *optional*, defaults to `default_value`): + A concise description for internal_helper_arg if not defined or overriding the description in `args_doc.py`. + """ +) +class MySpecialModel(PreTrainedModel): + def __init__(self, config: ConfigType, custom_parameter: "type" = "default_value", internal_helper_arg=None): + # ... +``` + +Or: + +```python +@auto_docstring( + custom_intro="""This model performs specific synergistic operations. + It builds upon the standard Transformer architecture with unique modifications.""", +) +class MySpecialModel(PreTrainedModel): + def __init__(self, config: ConfigType, custom_parameter: "type" = "default_value", internal_helper_arg=None): + r""" + custom_parameter (`type`, *optional*, defaults to `default_value`): + A concise description for custom_parameter if not defined or overriding the description in `args_doc.py`. + internal_helper_arg (`type`, *optional*, defaults to `default_value`): + A concise description for internal_helper_arg if not defined or overriding the description in `args_doc.py`. + """ + # ... +``` + +### 3. Applying to Functions (e.g., `forward` method) +Apply the decorator above method definitions, such as the `forward` method. + +```python + @auto_docstring + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + new_custom_argument: Optional[torch.Tensor] = None, + arg_documented_in_args_doc: Optional[torch.Tensor] = None, + # ... other arguments + ) -> Union[Tuple, ModelOutput]: # The description of the return value will automatically be generated from the ModelOutput class docstring. + r""" + new_custom_argument (`torch.Tensor`, *optional*): + Description of this new custom argument and its expected shape or type. + """ + # ... +``` + +#### Advanced Function Decoration: + +Arguments can be passed directly to `@auto_docstring` for more control. `Returns` and `Examples` sections can also be manually specified: + +```python +MODEL_COMMON_CUSTOM_ARGS = r""" + common_arg_1 (`torch.Tensor`, *optional*, defaults to `default_value`): + Description of common_arg_1 + common_arg_2 (`torch.Tensor`, *optional*, defaults to `default_value`): + Description of common_arg_2 + ... +""" + +class MyModel(PreTrainedModel): + # ... + @auto_docstring( + custom_intro=""" + This is a custom introduction for the function. + """ + custom_args=MODEL_COMMON_CUSTOM_ARGS + ) + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + common_arg_1: Optional[torch.Tensor] = None, + common_arg_2: Optional[torch.Tensor] = None, + #... + function_specific_argument: Optional[torch.Tensor] = None, + # ... other arguments + ) -> torch.Tensor: + r""" + function_specific_argument (`torch.Tensor`, *optional*): + Description of an argument specific to this function + + Returns: + `torch.Tensor`: For a function returning a generic type, a custom "Returns" section can be specified. + + Example: + + (To override the default example with a custom one or to add an example for a model class that does not have a pipeline) + + ```python + ... + ``` + """ + # ... +``` + +--- + +### ✍️ Documenting Arguments: Approach & Priority + +1. **Standard Arguments (e.g., `input_ids`, `attention_mask`, `pixel_values`, `encoder_hidden_states` etc.):** + * `@auto_docstring` retrieves descriptions from a central source. Do not redefine these locally if their description and shape are the same as in `args_doc.py`. + +2. **New or Custom Arguments:** + * **Primary Method:** Document these within an `r""" """` docstring block following the signature (for functions) or in the `__init__` method's docstring (for class parameters). + * **Format:** + ``` + argument_name (`type`, *optional*, defaults to `X`): + Description of the argument. + Explain its purpose, expected shape/type if complex, and default behavior. + This can span multiple lines. + ``` + * Include `type` in backticks. + * Add "*optional*" if the argument is not required (has a default value). + * Add "defaults to `X`" if it has a default value (no need to specify "defaults to `None`" if the default value is `None`). + +3. **Overriding Standard Arguments:** + * If a standard argument behaves differently (e.g., different expected shape, model-specific behavior), provide its complete description in the local `r""" """` docstring. This local definition takes precedence. + * The `labels` argument is often customized per model and typically requires a specific docstring. + +4. **Using Decorator Arguments for Overrides or New Arguments (`custom_args`):** + * New or custom arguments docstrings can also be passed to `@auto_docstring` as a `custom_args` argument. This can be used to define the docstring block for new arguments once if they are repeated in multiple places in the modeling file. + +--- + +### Usage with [modular files](./modular_transformers) + +When working with modular files, follow these guidelines for applying the `@auto_docstring` decorator: + +- **For standalone models in modular files:** + Apply the `@auto_docstring` decorator just as you would in regular modeling files. + +- **For models inheriting from other library models:** + - When inheriting from a parent model, decorators (including `@auto_docstring`) are automatically carried over to the generated modeling file without needing to add them in your modular file. + - If you need to modify the `@auto_docstring` behavior, apply the customized decorator in your modular file, making sure to *include all other decorators* that were present on the original function/class. + + > **Warning**: When overriding any decorator in a modular file, you must include ALL decorators that were applied to that function/class in the parent model. If you only override some decorators, the others won't be included in the generated modeling file. + + +**Note**: The `check_auto_docstrings` tool doesn't check modular files directly, but it will check (and modify when using `--fix_and_overwrite`) the generated modeling files. If issues are found in the generated files, you'll need to update your modular files accordingly. + +--- + +## ✅ Checking Your Docstrings with `check_auto_docstrings` + +The library includes a utility script to validate docstrings. This check is typically run during Continuous Integration (CI). + +#### What it Checks: + +* **Decorator Presence:** Ensures `@auto_docstring` is applied to relevant model classes and public methods. (TODO) +* **Argument Completeness & Consistency:** + * Flags arguments in the signature that are not known standard arguments and lack a local description. + * Ensures documented arguments exist in the signature. (TODO) + * Verifies that types and default values in the docstring match the signature. (TODO) +* **Placeholder Detection:** Reminds you to complete placeholders like `` or ``. +* **Formatting:** Adherence to the expected docstring style. + +#### Running the Check Locally: + +Run this check locally before committing. The common command is: + +```bash +make fix-copies +``` + +Alternatively, to only perform docstrings and auto-docstring checks, you can use: + +```bash +python utils/check_docstrings.py # to only check files included in the diff without fixing them +# Or: python utils/check_docstrings.py --fix_and_overwrite # to fix and overwrite the files in the diff +# Or: python utils/check_docstrings.py --fix_and_overwrite --check_all # to fix and overwrite all files +``` + +#### Workflow with the Checker: + +1. Add `@auto_docstring(...)` to the class or method. +2. For new, custom, or overridden arguments, add descriptions in an `r""" """` block. +3. Run `make fix-copies` (or the `check_docstrings.py` utility). + * For unrecognized arguments lacking documentation, the utility will create placeholder entries. +4. Manually edit these placeholders with accurate types and descriptions. +5. Re-run the check to ensure all issues are resolved. + +--- + +## 🔑 Key Takeaways & Best Practices + +* Use `@auto_docstring` for new PyTorch model classes (`PreTrainedModel` subclasses) and their primary for methods (e.g., `forward`, `get_text_features` etc.). +* For classes, the `__init__` method's docstring is the main source for parameter descriptions when using `@auto_docstring` on the class. +* Rely on standard docstrings; do not redefine common arguments unless their behavior is different in your specific model. +* Document new or custom arguments clearly. +* Run `check_docstrings` locally and iteratively. + +By following these guidelines, you help maintain consistent and informative documentation for the Hugging Face Transformers library 🤗. diff --git a/src/transformers/commands/add_fast_image_processor.py b/src/transformers/commands/add_fast_image_processor.py index dff3cb2d7b..3122fcee2c 100644 --- a/src/transformers/commands/add_fast_image_processor.py +++ b/src/transformers/commands/add_fast_image_processor.py @@ -396,9 +396,7 @@ def add_fast_image_processor_file( content_header = get_fast_image_processing_content_header(content_base_file) content_base_file = ( - f"@add_start_docstrings(\n" - f' "Constructs a fast {fast_image_processor_name.replace("ImageProcessorFast", "")} image processor.",\n' - f" BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,\n)\n" + f"@auto_docstring(\n" f"class {fast_image_processor_name}(BaseImageProcessorFast):\n" " # This generated class can be used as a starting point for the fast image processor.\n" " # if the image processor is only used for simple augmentations, such as resizing, center cropping, rescaling, or normalizing,\n" @@ -422,9 +420,7 @@ def add_fast_image_processor_file( f'__all__ = ["{fast_image_processor_name}"]\n' ) - imports = ( - "\n\nfrom ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast\n" - ) + imports = "\n\nfrom ...image_processing_utils_fast import BaseImageProcessorFast\n" image_utils_imports = [] if default_args_dict.get("resample") is not None and "PILImageResampling" in default_args_dict.get("resample"): image_utils_imports.append("PILImageResampling") @@ -442,7 +438,7 @@ def add_fast_image_processor_file( image_utils_imports.sort() imports += f"from ...image_utils import {', '.join(image_utils_imports)}\n" - imports += "from ...utils import add_start_docstrings\n" + imports += "from ...utils import auto_docstring\n" content = content_header + imports + "\n\n" + content_base_file diff --git a/src/transformers/image_processing_utils_fast.py b/src/transformers/image_processing_utils_fast.py index 2f590bce0e..064122bfa7 100644 --- a/src/transformers/image_processing_utils_fast.py +++ b/src/transformers/image_processing_utils_fast.py @@ -46,7 +46,7 @@ from .image_utils import ( from .processing_utils import Unpack from .utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -190,107 +190,7 @@ class DefaultFastImageProcessorKwargs(TypedDict, total=False): device: Optional["torch.device"] -BASE_IMAGE_PROCESSOR_FAST_DOCSTRING = r""" - - Args: - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the - `do_resize` parameter in the `preprocess` method. - size (`dict`, *optional*, defaults to `self.size`): - Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` - method. - default_to_square (`bool`, *optional*, defaults to `self.default_to_square`): - Whether to default to a square image when resizing, if size is an int. - resample (`PILImageResampling`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be - overridden by the `resample` parameter in the `preprocess` method. - do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): - Whether to center crop the image to the specified `crop_size`. Can be overridden by `do_center_crop` in the - `preprocess` method. - crop_size (`Dict[str, int]` *optional*, defaults to `self.crop_size`): - Size of the output image after applying `center_crop`. Can be overridden by `crop_size` in the `preprocess` - method. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the - `do_rescale` parameter in the `preprocess` method. - rescale_factor (`int` or `float`, *optional*, defaults to `self.rescale_factor`): - Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be - overridden by the `rescale_factor` parameter in the `preprocess` method. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` - method. Can be overridden by the `do_normalize` parameter in the `preprocess` method. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): - Mean to use if normalizing the image. This is a float or list of floats the length of the number of - channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be - overridden by the `image_mean` parameter in the `preprocess` method. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): - Standard deviation to use if normalizing the image. This is a float or list of floats the length of the - number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. - Can be overridden by the `image_std` parameter in the `preprocess` method. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*, defaults to `self.return_tensors`): - Returns stacked tensors if set to `pt, otherwise returns a list of tensors. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.data_format`): - Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors. - input_data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.input_data_format`): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - device (`torch.device`, *optional*, defaults to `self.device`): - The device to process the images on. If unset, the device is inferred from the input images.""" - -BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS = r""" - Preprocess an image or batch of images. - - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - size (`Dict[str, int]`, *optional*, defaults to `self.size`): - Describes the maximum input dimensions to the model. - resample (`PILImageResampling` or `InterpolationMode`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only - has an effect if `do_resize` is set to `True`. - do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): - Whether to center crop the image. - crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`): - Size of the output image after applying `center_crop`. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Rescale factor to rescale the image by if `do_rescale` is set to `True`. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): - Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): - Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to - `True`. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*, defaults to `self.return_tensors`): - Returns stacked tensors if set to `pt, otherwise returns a list of tensors. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.data_format`): - Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors. - input_data_format (`ChannelDimension` or `str`, *optional*, defaults to `self.input_data_format`): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - device (`torch.device`, *optional*, defaults to `self.device`): - The device to process the images on. If unset, the device is inferred from the input images.""" - - -@add_start_docstrings( - "Constructs a fast base image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class BaseImageProcessorFast(BaseImageProcessor): resample = None image_mean = None @@ -666,7 +566,7 @@ class BaseImageProcessorFast(BaseImageProcessor): data_format=data_format, ) - @add_start_docstrings(BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[DefaultFastImageProcessorKwargs]) -> BatchFeature: validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys()) # Set default kwargs from self. This ensures that if a kwarg is not provided diff --git a/src/transformers/models/albert/modeling_albert.py b/src/transformers/models/albert/modeling_albert.py index 68abb317ee..b896b9646b 100755 --- a/src/transformers/models/albert/modeling_albert.py +++ b/src/transformers/models/albert/modeling_albert.py @@ -41,22 +41,12 @@ from ...pytorch_utils import ( is_torch_greater_or_equal_than_2_2, prune_linear_layer, ) -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_albert import AlbertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "albert/albert-base-v2" -_CONFIG_FOR_DOC = "AlbertConfig" - def load_tf_weights_in_albert(model, config, tf_checkpoint_path): """Load tf checkpoints in a pytorch model.""" @@ -553,12 +543,8 @@ class AlbertTransformer(nn.Module): ) +@auto_docstring class AlbertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = AlbertConfig load_tf_weights = load_tf_weights_in_albert base_model_prefix = "albert" @@ -617,81 +603,16 @@ class AlbertForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -ALBERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Args: - config ([`AlbertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ALBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and - [`PreTrainedTokenizer.encode`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", - ALBERT_START_DOCSTRING, -) +@auto_docstring class AlbertModel(AlbertPreTrainedModel): config_class = AlbertConfig base_model_prefix = "albert" def __init__(self, config: AlbertConfig, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -733,12 +654,7 @@ class AlbertModel(AlbertPreTrainedModel): inner_group_idx = int(layer - group_idx * self.config.inner_group_num) self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -826,12 +742,11 @@ class AlbertModel(AlbertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Albert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `sentence order prediction (classification)` head. - """, - ALBERT_START_DOCSTRING, + """ ) class AlbertForPreTraining(AlbertPreTrainedModel): _tied_weights_keys = ["predictions.decoder.bias", "predictions.decoder.weight"] @@ -855,8 +770,7 @@ class AlbertForPreTraining(AlbertPreTrainedModel): def get_input_embeddings(self) -> nn.Embedding: return self.albert.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=AlbertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -881,8 +795,6 @@ class AlbertForPreTraining(AlbertPreTrainedModel): (see `input_ids` docstring) Indices should be in `[0, 1]`. `0` indicates original order (sequence A, then sequence B), `1` indicates switched order (sequence B, then sequence A). - Returns: - Example: ```python @@ -981,10 +893,7 @@ class AlbertSOPHead(nn.Module): return logits -@add_start_docstrings( - "Albert Model with a `language modeling` head on top.", - ALBERT_START_DOCSTRING, -) +@auto_docstring class AlbertForMaskedLM(AlbertPreTrainedModel): _tied_weights_keys = ["predictions.decoder.bias", "predictions.decoder.weight"] @@ -1007,8 +916,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def get_input_embeddings(self) -> nn.Embedding: return self.albert.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1028,8 +936,6 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - Returns: - Example: ```python @@ -1093,12 +999,11 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ALBERT_START_DOCSTRING, + """ ) class AlbertForSequenceClassification(AlbertPreTrainedModel): def __init__(self, config: AlbertConfig): @@ -1113,14 +1018,7 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="textattack/albert-base-v2-imdb", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'LABEL_1'", - expected_loss=0.12, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1194,13 +1092,7 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): ) -@add_start_docstrings( - """ - Albert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - ALBERT_START_DOCSTRING, -) +@auto_docstring class AlbertForTokenClassification(AlbertPreTrainedModel): def __init__(self, config: AlbertConfig): super().__init__(config) @@ -1218,12 +1110,7 @@ class AlbertForTokenClassification(AlbertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1277,13 +1164,7 @@ class AlbertForTokenClassification(AlbertPreTrainedModel): ) -@add_start_docstrings( - """ - Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - ALBERT_START_DOCSTRING, -) +@auto_docstring class AlbertForQuestionAnswering(AlbertPreTrainedModel): def __init__(self, config: AlbertConfig): super().__init__(config) @@ -1295,16 +1176,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="twmkn9/albert-base-v2-squad2", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - qa_target_start_index=12, - qa_target_end_index=13, - expected_output="'a nice puppet'", - expected_loss=7.36, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1319,16 +1191,6 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[AlbertForPreTrainingOutput, Tuple]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.albert( @@ -1380,13 +1242,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel): ) -@add_start_docstrings( - """ - Albert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - ALBERT_START_DOCSTRING, -) +@auto_docstring class AlbertForMultipleChoice(AlbertPreTrainedModel): def __init__(self, config: AlbertConfig): super().__init__(config) @@ -1398,12 +1254,7 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1418,6 +1269,30 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[AlbertForPreTrainingOutput, Tuple]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + [`PreTrainedTokenizer.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index b761615ac2..bdebd31a26 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -31,154 +31,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_align import AlignConfig, AlignTextConfig, AlignVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "kakaobrain/align-base" -_CONFIG_FOR_DOC = "AlignConfig" - - -ALIGN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`AlignConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ALIGN_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -ALIGN_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`EfficientNetImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -ALIGN_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`EfficientNetImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - @dataclass class AlignVisionModelOutput(ModelOutput): @@ -1165,12 +1023,8 @@ class AlignTextPooler(nn.Module): return pooled_output +@auto_docstring class AlignPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = AlignConfig base_model_prefix = "align" supports_gradient_checkpointing = True @@ -1194,15 +1048,20 @@ class AlignPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -@add_start_docstrings( - """The text model from ALIGN without any head or projection on top.""", - ALIGN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text model from ALIGN without any head or projection on top. + """ ) class AlignTextModel(AlignPreTrainedModel): config_class = AlignTextConfig _no_split_modules = ["AlignTextEmbeddings"] def __init__(self, config: AlignTextConfig, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1220,8 +1079,7 @@ class AlignTextModel(AlignPreTrainedModel): def set_input_embeddings(self, value): self.embeddings.word_embeddings = value - @add_start_docstrings_to_model_forward(ALIGN_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=AlignTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1235,8 +1093,6 @@ class AlignTextModel(AlignPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: r""" - Returns: - Examples: ```python @@ -1321,9 +1177,10 @@ class AlignTextModel(AlignPreTrainedModel): ) -@add_start_docstrings( - """The vision model from ALIGN without any head or projection on top.""", - ALIGN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from ALIGN without any head or projection on top. + """ ) class AlignVisionModel(AlignPreTrainedModel): config_class = AlignVisionConfig @@ -1350,8 +1207,7 @@ class AlignVisionModel(AlignPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.convolution - @add_start_docstrings_to_model_forward(ALIGN_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPoolingAndNoAttention, config_class=AlignVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1359,8 +1215,6 @@ class AlignVisionModel(AlignPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndNoAttention]: r""" - Returns: - Examples: ```python @@ -1410,7 +1264,7 @@ class AlignVisionModel(AlignPreTrainedModel): ) -@add_start_docstrings(ALIGN_START_DOCSTRING) +@auto_docstring class AlignModel(AlignPreTrainedModel): config_class = AlignConfig @@ -1444,7 +1298,7 @@ class AlignModel(AlignPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ALIGN_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1497,7 +1351,7 @@ class AlignModel(AlignPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(ALIGN_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1542,8 +1396,7 @@ class AlignModel(AlignPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(ALIGN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=AlignOutput, config_class=AlignConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1559,7 +1412,8 @@ class AlignModel(AlignPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, AlignOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: diff --git a/src/transformers/models/altclip/modeling_altclip.py b/src/transformers/models/altclip/modeling_altclip.py index 9c8be7ea80..5637bc6dee 100755 --- a/src/transformers/models/altclip/modeling_altclip.py +++ b/src/transformers/models/altclip/modeling_altclip.py @@ -32,125 +32,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_altclip import AltCLIPConfig, AltCLIPTextConfig, AltCLIPVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "BAAI/AltCLIP" -_CONFIG_FOR_DOC = "AltCLIPConfig" - - -ALTCLIP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`CLIPConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ALTCLIP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -ALTCLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -ALTCLIP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # contrastive loss function, adapted from # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/CLIP.html @@ -1076,12 +963,8 @@ class AltCLIPVisionEmbeddings(nn.Module): return embeddings +@auto_docstring class AltCLIPPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = AltCLIPConfig base_model_prefix = "altclip" supports_gradient_checkpointing = True @@ -1144,8 +1027,7 @@ class AltCLIPVisionTransformer(nn.Module): self.encoder = AltCLIPEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(ALTCLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=AltCLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1154,10 +1036,6 @@ class AltCLIPVisionTransformer(nn.Module): return_dict: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = False, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1205,8 +1083,7 @@ class AltCLIPVisionModel(AltCLIPPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(ALTCLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=AltCLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1216,8 +1093,6 @@ class AltCLIPVisionModel(AltCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -1248,9 +1123,8 @@ class AltCLIPVisionModel(AltCLIPPreTrainedModel): ) -class AltRobertaModel(AltCLIPPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz @@ -1261,13 +1135,17 @@ class AltRobertaModel(AltCLIPPreTrainedModel): `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 - """ - +) +class AltRobertaModel(AltCLIPPreTrainedModel): config_class = AltCLIPTextConfig # Copied from transformers.models.clap.modeling_clap.ClapTextModel.__init__ with ClapText->AltRoberta def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1293,6 +1171,7 @@ class AltRobertaModel(AltCLIPPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) + @auto_docstring # Copied from transformers.models.clap.modeling_clap.ClapTextModel.forward def forward( self, @@ -1310,26 +1189,6 @@ class AltRobertaModel(AltCLIPPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1444,8 +1303,7 @@ class AltCLIPTextModel(AltCLIPPreTrainedModel): def resize_token_embeddings(self, new_num_tokens: Optional[int] = None) -> nn.Embedding: return super().resize_token_embeddings(new_num_tokens) - @add_start_docstrings_to_model_forward(ALTCLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPoolingAndProjection, config_class=AltCLIPTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1461,8 +1319,6 @@ class AltCLIPTextModel(AltCLIPPreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndProjection]: r""" - Returns: - Examples: ```python @@ -1551,7 +1407,7 @@ class AltCLIPModel(AltCLIPPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ALTCLIP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1598,7 +1454,7 @@ class AltCLIPModel(AltCLIPPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(ALTCLIP_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1646,8 +1502,7 @@ class AltCLIPModel(AltCLIPPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(ALTCLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=AltCLIPOutput, config_class=AltCLIPConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1662,7 +1517,8 @@ class AltCLIPModel(AltCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, AltCLIPOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index bef0e2cec1..9a02e1c58d 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -34,12 +34,10 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import ( LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging, - replace_return_docstrings, ) from ...utils.import_utils import is_torch_available from ..auto import AutoModel @@ -60,9 +58,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "AriaConfig" - - @use_kernel_forward_from_hub("RMSNorm") class AriaTextRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -656,11 +651,8 @@ class AriaTextDecoderLayer(GradientCheckpointingLayer): return outputs +@auto_docstring class AriaTextPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. - """ - config_class = AriaTextConfig base_model_prefix = "model" _no_split_modules = ["AriaTextDecoderLayer", "AriaGroupedExpertsGemm"] @@ -687,27 +679,7 @@ class AriaTextPreTrainedModel(PreTrainedModel): module.weight.data.normal_(mean=0.0, std=std) -ARIA_TEXT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`AriaTextConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Aria Model outputting raw hidden-states without any specific head on top.", - ARIA_TEXT_START_DOCSTRING, -) +@auto_docstring class AriaPreTrainedModel(PreTrainedModel): config_class = AriaConfig base_model_prefix = "" @@ -773,88 +745,8 @@ class AriaTextRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -ARIA_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare AriaText Model outputting raw hidden-states without any specific head on top.", - ARIA_TEXT_START_DOCSTRING, -) +@auto_docstring class AriaTextModel(AriaTextPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`AriaTextDecoderLayer`] - - Args: - config: AriaTextConfig - """ - def __init__(self, config: AriaTextConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -878,7 +770,7 @@ class AriaTextModel(AriaTextPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(ARIA_TEXT_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1100,21 +992,8 @@ class AriaTextModel(AriaTextPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -_CONFIG_FOR_TEXT_DOC = "AriaTextConfig" - - +@auto_docstring class AriaTextForCausalLM(AriaTextPreTrainedModel, GenerationMixin): - """ - Aria model for causal language modeling tasks. - - This class extends `LlamaForCausalLM` to incorporate the Mixture of Experts (MoE) approach, - allowing for more efficient and scalable language modeling. - - Args: - config (`AriaTextConfig`): - Configuration object for the model. - """ - _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} @@ -1146,8 +1025,7 @@ class AriaTextForCausalLM(AriaTextPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(ARIA_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_TEXT_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1164,19 +1042,10 @@ class AriaTextForCausalLM(AriaTextPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1304,61 +1173,10 @@ class AriaModelOutputWithPast(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None -ARIA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor`, *optional*): - Input token IDs. - pixel_values (`torch.FloatTensor`, *optional*): - Pixel values of the images. - pixel_mask (`torch.LongTensor`, *optional*): - Mask for the pixel values. - attention_mask (`torch.Tensor`, *optional*): - Attention mask. - position_ids (`torch.LongTensor`, *optional*): - Position IDs. - past_key_values (`List[torch.FloatTensor]`, *optional*): - Past key values for efficient processing. - inputs_embeds (`torch.FloatTensor`, *optional*): - Input embeddings. - labels (`torch.LongTensor`, *optional*): - Labels for computing the language modeling loss. - use_cache (`bool`, *optional*): - Whether to use the model's cache mechanism. - output_attentions (`bool`, *optional*): - Whether to output attention weights. - output_hidden_states (`bool`, *optional*): - Whether to output hidden states. - return_dict (`bool`, *optional*): - Whether to return a `ModelOutput` object. - logits_to_keep (`int` or `torch.Tensor`, *optional*, defaults to 0): - If an `int`, calculate logits for the last `logits_to_keep` tokens, or all `input_ids` if `0`. - Otherwise, slice according to the 1D tensor in the sequence length dimension - cache_position (`torch.LongTensor`, *optional*): - Cache positions. - **loss_kwargs: - Additional keyword arguments for loss calculation. -""" - -ARIA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config (`AriaConfig`): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The Aria model which consists of a vision backbone and a language model, without a language modeling head.""", - ARIA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Aria model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class AriaModel(AriaPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -1411,7 +1229,7 @@ class AriaModel(AriaPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(ARIA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1501,11 +1319,13 @@ class AriaModel(AriaPreTrainedModel): return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() -@add_start_docstrings( - """Aria model for conditional generation tasks. +@auto_docstring( + custom_intro=""" + Aria model for conditional generation tasks. + This model combines a vision tower, a multi-modal projector, and a language model - to perform tasks that involve both image and text inputs.""", - ARIA_START_DOCSTRING, + to perform tasks that involve both image and text inputs. + """ ) class AriaForConditionalGeneration(AriaPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -1548,8 +1368,7 @@ class AriaForConditionalGeneration(AriaPreTrainedModel, GenerationMixin): return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(ARIA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=AriaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1569,12 +1388,11 @@ class AriaForConditionalGeneration(AriaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, AriaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics3ForConditionalGeneration`). - Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `AriaForConditionalGeneration`). + Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only + computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index e6beaf8919..c741a9b2c4 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -33,22 +33,10 @@ from ...image_utils import ( validate_preprocess_arguments, ) from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack -from ...tokenization_utils import ( - PreTokenizedInput, - TextInput, -) -from ...utils import ( - LossKwargs, - TensorType, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...tokenization_utils import PreTokenizedInput, TextInput +from ...utils import LossKwargs, TensorType, auto_docstring, can_return_tuple, logging from ...utils.import_utils import is_torch_available from ..auto import CONFIG_MAPPING, AutoConfig, AutoTokenizer from ..llama.configuration_llama import LlamaConfig @@ -76,11 +64,6 @@ if is_torch_available(): from torch import nn -_CONFIG_FOR_DOC = "AriaConfig" -_CONFIG_FOR_TEXT_DOC = "AriaTextConfig" -ARIA_TEXT_INPUTS_DOCSTRING = None - - def sequential_experts_gemm(token_states, expert_weights, tokens_per_expert): """ Compute the matrix multiplication (GEMM) for each expert sequentially. This approach is computationally inefficient, especially when dealing with a large number of experts. @@ -1229,11 +1212,8 @@ class AriaTextDecoderLayer(LlamaDecoderLayer): self.mlp = AriaTextMoELayer(config) +@auto_docstring class AriaTextPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. - """ - config_class = AriaTextConfig base_model_prefix = "model" _no_split_modules = ["AriaTextDecoderLayer", "AriaGroupedExpertsGemm"] @@ -1297,17 +1277,6 @@ class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... class AriaTextForCausalLM(AriaTextPreTrainedModel, LlamaForCausalLM): - """ - Aria model for causal language modeling tasks. - - This class extends `LlamaForCausalLM` to incorporate the Mixture of Experts (MoE) approach, - allowing for more efficient and scalable language modeling. - - Args: - config (`AriaTextConfig`): - Configuration object for the model. - """ - _tied_weights_keys = ["lm_head.weight"] def __init__(self, config: AriaTextConfig): @@ -1319,8 +1288,7 @@ class AriaTextForCausalLM(AriaTextPreTrainedModel, LlamaForCausalLM): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ARIA_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_TEXT_DOC) + @auto_docstring def forward(self, **super_kwargs): super().forward(self, **super_kwargs) @@ -1333,58 +1301,6 @@ class AriaModelOutputWithPast(LlavaModelOutputWithPast): pass -ARIA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor`, *optional*): - Input token IDs. - pixel_values (`torch.FloatTensor`, *optional*): - Pixel values of the images. - pixel_mask (`torch.LongTensor`, *optional*): - Mask for the pixel values. - attention_mask (`torch.Tensor`, *optional*): - Attention mask. - position_ids (`torch.LongTensor`, *optional*): - Position IDs. - past_key_values (`List[torch.FloatTensor]`, *optional*): - Past key values for efficient processing. - inputs_embeds (`torch.FloatTensor`, *optional*): - Input embeddings. - labels (`torch.LongTensor`, *optional*): - Labels for computing the language modeling loss. - use_cache (`bool`, *optional*): - Whether to use the model's cache mechanism. - output_attentions (`bool`, *optional*): - Whether to output attention weights. - output_hidden_states (`bool`, *optional*): - Whether to output hidden states. - return_dict (`bool`, *optional*): - Whether to return a `ModelOutput` object. - logits_to_keep (`int` or `torch.Tensor`, *optional*, defaults to 0): - If an `int`, calculate logits for the last `logits_to_keep` tokens, or all `input_ids` if `0`. - Otherwise, slice according to the 1D tensor in the sequence length dimension - cache_position (`torch.LongTensor`, *optional*): - Cache positions. - **loss_kwargs: - Additional keyword arguments for loss calculation. -""" - -ARIA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config (`AriaConfig`): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - class AriaModel(LlavaModel): def __init__(self, config: AriaConfig): super().__init__(config) @@ -1440,8 +1356,6 @@ class AriaModel(LlavaModel): image_features = self.multi_modal_projector(selected_image_feature, attn_mask=image_attn_mask) return image_features - @can_return_tuple - @add_start_docstrings_to_model_forward(ARIA_INPUTS_DOCSTRING) def forward( self, input_ids: torch.LongTensor = None, @@ -1515,16 +1429,17 @@ class AriaModel(LlavaModel): ) -@add_start_docstrings( - """Aria model for conditional generation tasks. +@auto_docstring( + custom_intro=""" + Aria model for conditional generation tasks. + This model combines a vision tower, a multi-modal projector, and a language model - to perform tasks that involve both image and text inputs.""", - ARIA_START_DOCSTRING, + to perform tasks that involve both image and text inputs. + """ ) class AriaForConditionalGeneration(LlavaForConditionalGeneration): @can_return_tuple - @add_start_docstrings_to_model_forward(ARIA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=AriaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1544,12 +1459,11 @@ class AriaForConditionalGeneration(LlavaForConditionalGeneration): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, AriaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics3ForConditionalGeneration`). - Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `AriaForConditionalGeneration`). + Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only + computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py b/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py index 1aa24ca08a..90fc336215 100644 --- a/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py +++ b/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py @@ -25,24 +25,12 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, SequenceClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_audio_spectrogram_transformer import ASTConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "ASTConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "MIT/ast-finetuned-audioset-10-10-0.4593" -_EXPECTED_OUTPUT_SHAPE = [1, 1214, 768] - -# Audio classification docstring -_SEQ_CLASS_CHECKPOINT = "MIT/ast-finetuned-audioset-10-10-0.4593" -_SEQ_CLASS_EXPECTED_OUTPUT = "'Speech'" -_SEQ_CLASS_EXPECTED_LOSS = 0.17 - class ASTEmbeddings(nn.Module): """ @@ -388,12 +376,8 @@ class ASTEncoder(nn.Module): ) +@auto_docstring class ASTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ASTConfig base_model_prefix = "audio_spectrogram_transformer" main_input_name = "input_values" @@ -420,48 +404,7 @@ class ASTPreTrainedModel(PreTrainedModel): module.distillation_token.data.zero_() -AUDIO_SPECTROGRAM_TRANSFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ASTConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -AUDIO_SPECTROGRAM_TRANSFORMER_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, max_length, num_mel_bins)`): - Float values mel features extracted from the raw audio waveform. Raw audio waveform can be obtained by - loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via - the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~ASTFeatureExtractor.__call__`] - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare AST Model transformer outputting raw hidden-states without any specific head on top.", - AUDIO_SPECTROGRAM_TRANSFORMER_START_DOCSTRING, -) +@auto_docstring class ASTModel(ASTPreTrainedModel): def __init__(self, config: ASTConfig) -> None: super().__init__(config) @@ -486,14 +429,7 @@ class ASTModel(ASTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(AUDIO_SPECTROGRAM_TRANSFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor] = None, @@ -502,6 +438,14 @@ class ASTModel(ASTPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + input_values (`torch.FloatTensor` of shape `(batch_size, max_length, num_mel_bins)`): + Float values mel features extracted from the raw audio waveform. Raw audio waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~ASTFeatureExtractor.__call__`] + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -555,12 +499,11 @@ class ASTMLPHead(nn.Module): return hidden_state -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Audio Spectrogram Transformer model with an audio classification head on top (a linear layer on top of the pooled output) e.g. for datasets like AudioSet, Speech Commands v2. - """, - AUDIO_SPECTROGRAM_TRANSFORMER_START_DOCSTRING, + """ ) class ASTForAudioClassification(ASTPreTrainedModel): def __init__(self, config: ASTConfig) -> None: @@ -575,15 +518,7 @@ class ASTForAudioClassification(ASTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(AUDIO_SPECTROGRAM_TRANSFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_SEQ_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor] = None, @@ -594,6 +529,12 @@ class ASTForAudioClassification(ASTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, max_length, num_mel_bins)`): + Float values mel features extracted from the raw audio waveform. Raw audio waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~ASTFeatureExtractor.__call__`] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the audio classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/autoformer/modeling_autoformer.py b/src/transformers/models/autoformer/modeling_autoformer.py index 8a4e9cd8e4..8cb92caa1a 100644 --- a/src/transformers/models/autoformer/modeling_autoformer.py +++ b/src/transformers/models/autoformer/modeling_autoformer.py @@ -27,22 +27,15 @@ from torch import nn from ...activations import ACT2FN from ...modeling_attn_mask_utils import _prepare_4d_attention_mask -from ...modeling_outputs import ( - BaseModelOutput, - ModelOutput, - SampleTSPredictionOutput, - Seq2SeqTSPredictionOutput, -) +from ...modeling_outputs import BaseModelOutput, ModelOutput, SampleTSPredictionOutput, Seq2SeqTSPredictionOutput from ...modeling_utils import PreTrainedModel from ...time_series_utils import NegativeBinomialOutput, NormalOutput, StudentTOutput -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_autoformer import AutoformerConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "AutoformerConfig" - @dataclass class AutoFormerDecoderOutput(ModelOutput): @@ -888,6 +881,7 @@ class AutoformerDecoderLayer(nn.Module): return outputs +@auto_docstring class AutoformerPreTrainedModel(PreTrainedModel): config_class = AutoformerConfig base_model_prefix = "model" @@ -908,154 +902,6 @@ class AutoformerPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -AUTOFORMER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`AutoformerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -AUTOFORMER_INPUTS_DOCSTRING = r""" - Args: - past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Past values of the time series, that serve as context in order to predict the future. These values may - contain lags, i.e. additional values from the past which are added in order to serve as "extra context". - The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as - `static_categorical_features`, `static_real_features`, `past_time_features`). - - The sequence length here is equal to `context_length` + `max(config.lags_sequence)`. - - Missing values need to be replaced with zeros. - - past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`, *optional*): - Optional time features, which the model internally will add to `past_values`. These could be things like - "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These - could also be so-called "age" features, which basically help the model know "at which point in life" a - time-series is. Age features have small values for distant past time steps and increase monotonically the - more we approach the current time step. - - These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where - the position encodings are learned from scratch internally as parameters of the model, the Time Series - Transformer requires to provide additional time features. - - The Autoformer only learns additional embeddings for `static_categorical_features`. - - past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): - Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in - `[0, 1]`: - - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - - static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): - Optional static categorical features for which the model will learn an embedding, which it will add to the - values of the time series. - - Static categorical features are features which have the same value for all time steps (static over time). - - A typical example of a static categorical feature is a time series ID. - - static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): - Optional static real features which the model will add to the values of the time series. - - Static real features are features which have the same value for all time steps (static over time). - - A typical example of a static real feature is promotion information. - - future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)`): - Future values of the time series, that serve as labels for the model. The `future_values` is what the - Transformer needs to learn to output, given the `past_values`. - - See the demo notebook and code snippets for details. - - Missing values need to be replaced with zeros. - - future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`, *optional*): - Optional time features, which the model internally will add to `future_values`. These could be things like - "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These - could also be so-called "age" features, which basically help the model know "at which point in life" a - time-series is. Age features have small values for distant past time steps and increase monotonically the - more we approach the current time step. - - These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where - the position encodings are learned from scratch internally as parameters of the model, the Time Series - Transformer requires to provide additional features. - - The Autoformer only learns additional embeddings for `static_categorical_features`. - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on certain token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Mask to avoid performing attention on certain token indices. By default, a causal mask will be used, to - make sure the model can only look at previous inputs in order to predict the future. - - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesTransformerEncoder with TimeSeriesTransformer->Autoformer,TimeSeries->Autoformer class AutoformerEncoder(AutoformerPreTrainedModel): """ @@ -1417,10 +1263,7 @@ class AutoformerDecoder(AutoformerPreTrainedModel): ) -@add_start_docstrings( - "The bare Autoformer Model outputting raw hidden-states without any specific head on top.", - AUTOFORMER_START_DOCSTRING, -) +@auto_docstring class AutoformerModel(AutoformerPreTrainedModel): def __init__(self, config: AutoformerConfig): super().__init__(config) @@ -1595,8 +1438,7 @@ class AutoformerModel(AutoformerPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(AUTOFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=AutoformerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1618,7 +1460,74 @@ class AutoformerModel(AutoformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[AutoformerModelOutput, Tuple]: r""" - Returns: + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Past values of the time series, that serve as context in order to predict the future. These values may + contain lags, i.e. additional values from the past which are added in order to serve as "extra context". + The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as + `static_categorical_features`, `static_real_features`, `past_time_features`). + + The sequence length here is equal to `context_length` + `max(config.lags_sequence)`. + + Missing values need to be replaced with zeros. + past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`, *optional*): + Optional time features, which the model internally will add to `past_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. + + The Autoformer only learns additional embeddings for `static_categorical_features`. + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in + `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): + Optional static categorical features for which the model will learn an embedding, which it will add to the + values of the time series. + + Static categorical features are features which have the same value for all time steps (static over time). + + A typical example of a static categorical feature is a time series ID. + static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): + Optional static real features which the model will add to the values of the time series. + + Static real features are features which have the same value for all time steps (static over time). + + A typical example of a static real feature is promotion information. + future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)`): + Future values of the time series, that serve as labels for the model. The `future_values` is what the + Transformer needs to learn to output, given the `past_values`. + + See the demo notebook and code snippets for details. + + Missing values need to be replaced with zeros. + future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`, *optional*): + Optional time features, which the model internally will add to `future_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional features. + + The Autoformer only learns additional embeddings for `static_categorical_features`. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. Examples: @@ -1753,10 +1662,7 @@ class AutoformerModel(AutoformerPreTrainedModel): ) -@add_start_docstrings( - "The Autoformer Model with a distribution head on top for time-series forecasting.", - AUTOFORMER_START_DOCSTRING, -) +@auto_docstring class AutoformerForPrediction(AutoformerPreTrainedModel): def __init__(self, config: AutoformerConfig): super().__init__(config) @@ -1797,8 +1703,7 @@ class AutoformerForPrediction(AutoformerPreTrainedModel): sliced_params = [p[:, -trailing_n:] for p in params] return self.distribution_output.distribution(sliced_params, loc=loc, scale=scale) - @add_start_docstrings_to_model_forward(AUTOFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqTSPredictionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1821,7 +1726,82 @@ class AutoformerForPrediction(AutoformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Seq2SeqTSPredictionOutput, Tuple]: r""" - Returns: + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Past values of the time series, that serve as context in order to predict the future. These values may + contain lags, i.e. additional values from the past which are added in order to serve as "extra context". + The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as + `static_categorical_features`, `static_real_features`, `past_time_features`). + + The sequence length here is equal to `context_length` + `max(config.lags_sequence)`. + + Missing values need to be replaced with zeros. + past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`, *optional*): + Optional time features, which the model internally will add to `past_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. + + The Autoformer only learns additional embeddings for `static_categorical_features`. + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in + `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): + Optional static categorical features for which the model will learn an embedding, which it will add to the + values of the time series. + + Static categorical features are features which have the same value for all time steps (static over time). + + A typical example of a static categorical feature is a time series ID. + static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): + Optional static real features which the model will add to the values of the time series. + + Static real features are features which have the same value for all time steps (static over time). + + A typical example of a static real feature is promotion information. + future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)`): + Future values of the time series, that serve as labels for the model. The `future_values` is what the + Transformer needs to learn to output, given the `past_values`. + + See the demo notebook and code snippets for details. + + Missing values need to be replaced with zeros. + future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`, *optional*): + Optional time features, which the model internally will add to `future_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional features. + + The Autoformer only learns additional embeddings for `static_categorical_features`. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + future_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `future_values` were observed and which were missing. Mask values selected + in `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + + This mask is used to filter out missing values for the final loss calculation. Examples: diff --git a/src/transformers/models/aya_vision/modeling_aya_vision.py b/src/transformers/models/aya_vision/modeling_aya_vision.py index c700a97129..0e00263584 100644 --- a/src/transformers/models/aya_vision/modeling_aya_vision.py +++ b/src/transformers/models/aya_vision/modeling_aya_vision.py @@ -31,21 +31,11 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling from ..auto import AutoModel from .configuration_aya_vision import AyaVisionConfig -_CONFIG_FOR_DOC = "AyaVisionConfig" - - class AyaVisionMultiModalProjector(nn.Module): def __init__(self, config: AyaVisionConfig): super().__init__() @@ -96,27 +86,7 @@ class AyaVisionMultiModalProjector(nn.Module): return image_features -AYA_VISION_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`AyaVisionConfig`] or [`AyaVisionVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Aya Vision Model outputting raw hidden-states without any specific head on top.", - AYA_VISION_START_DOCSTRING, -) +@auto_docstring class AyaVisionPreTrainedModel(PreTrainedModel): config_class = AyaVisionConfig base_model_prefix = "" @@ -218,86 +188,10 @@ class AyaVisionModelOutputWithPast(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None -AYA_VISION_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`AyaVisionProcessor`] uses - [`CLIPImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The AyaVision model which consists of a vision backbone and a language model, without a language modeling head.""", - AYA_VISION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The AyaVision model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class AyaVisionModel(AyaVisionPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -363,7 +257,7 @@ class AyaVisionModel(AyaVisionPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(AYA_VISION_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -454,9 +348,10 @@ class AyaVisionModel(AyaVisionPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The AyaVision model which consists of a vision backbone and a language model.""", - AYA_VISION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The AYA_VISION model which consists of a vision backbone and a language model. + """ ) class AyaVisionForConditionalGeneration(AyaVisionPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -499,8 +394,7 @@ class AyaVisionForConditionalGeneration(AyaVisionPreTrainedModel, GenerationMixi return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(AYA_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=AyaVisionCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -522,20 +416,10 @@ class AyaVisionForConditionalGeneration(AyaVisionPreTrainedModel, GenerationMixi **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, AyaVisionCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/aya_vision/modular_aya_vision.py b/src/transformers/models/aya_vision/modular_aya_vision.py index 977170708f..7dcd206bab 100644 --- a/src/transformers/models/aya_vision/modular_aya_vision.py +++ b/src/transformers/models/aya_vision/modular_aya_vision.py @@ -29,17 +29,12 @@ from transformers.models.llava.modeling_llava import ( from ...activations import ACT2FN from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - logging, -) +from ...utils import logging from .configuration_aya_vision import AyaVisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "AyaVisionConfig" - class AyaVisionMultiModalProjector(nn.Module): def __init__(self, config: AyaVisionConfig): @@ -91,14 +86,6 @@ class AyaVisionMultiModalProjector(nn.Module): return image_features -AYA_VISION_START_DOCSTRING = None -AYA_VISION_INPUTS_DOCSTRING = None - - -@add_start_docstrings( - "The bare Aya Vision Model outputting raw hidden-states without any specific head on top.", - AYA_VISION_START_DOCSTRING, -) class AyaVisionPreTrainedModel(LlavaPreTrainedModel): _supports_quantized_cache = False _supports_static_cache = False @@ -127,10 +114,6 @@ class AyaVisionModel(LlavaModel): pass -@add_start_docstrings( - """The AyaVision model which consists of a vision backbone and a language model.""", - AYA_VISION_START_DOCSTRING, -) class AyaVisionForConditionalGeneration(LlavaForConditionalGeneration): def forward( self, @@ -153,20 +136,10 @@ class AyaVisionForConditionalGeneration(LlavaForConditionalGeneration): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, AyaVisionCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/bamba/modeling_bamba.py b/src/transformers/models/bamba/modeling_bamba.py index 7d172b55cc..8631688149 100644 --- a/src/transformers/models/bamba/modeling_bamba.py +++ b/src/transformers/models/bamba/modeling_bamba.py @@ -41,13 +41,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, logging from ...utils.import_utils import is_causal_conv1d_available, is_mamba_2_ssm_available from .configuration_bamba import BambaConfig @@ -66,8 +60,6 @@ else: logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "BambaConfig" - # Adapted from transformers.models.jamba.modeling_jamba.HybridMambaAttentionDynamicCache for the v2 mixer class HybridMambaAttentionDynamicCache(modeling_jamba.HybridMambaAttentionDynamicCache): @@ -1014,27 +1006,7 @@ class BambaDecoderLayer(nn.Module): return outputs -BAMBA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BambaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare BambaModel outputting raw hidden-states without any specific head on top.", - BAMBA_START_DOCSTRING, -) +@auto_docstring class BambaPreTrainedModel(PreTrainedModel): config_class = BambaConfig base_model_prefix = "model" @@ -1064,91 +1036,8 @@ class BambaPreTrainedModel(PreTrainedModel): module.D.data.fill_(1.0) -BAMBA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`HybridMambaAttentionDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - A HybridMambaAttentionDynamicCache object containing pre-computed hidden-states (keys and values in the - self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`. - Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and - `(batch_size, d_inner, d_state)` respectively. - See the `HybridMambaAttentionDynamicCache` class for more details. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Bamba Model outputting raw hidden-states without any specific head on top.", - BAMBA_START_DOCSTRING, -) -# Adapted from transformers.models.jamba.modeling_jamba.JambaModel +@auto_docstring class BambaModel(BambaPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`BambaDecoderLayer`] - - Args: - config: BambaConfig - """ - def __init__(self, config: BambaConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1175,7 +1064,7 @@ class BambaModel(BambaPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(BAMBA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1414,6 +1303,7 @@ class BambaModel(BambaPreTrainedModel): return mamba_mask +@auto_docstring class BambaForCausalLM(BambaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -1447,8 +1337,7 @@ class BambaForCausalLM(BambaPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(BAMBA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1465,19 +1354,10 @@ class BambaForCausalLM(BambaPreTrainedModel, GenerationMixin): **kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/bamba/modular_bamba.py b/src/transformers/models/bamba/modular_bamba.py index d6530747ae..954dfd11ab 100644 --- a/src/transformers/models/bamba/modular_bamba.py +++ b/src/transformers/models/bamba/modular_bamba.py @@ -43,23 +43,11 @@ from transformers.models.mamba2.modeling_mamba2 import ( segment_sum, ) -from ...modeling_attn_mask_utils import ( - AttentionMaskConverter, -) +from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) -from ...utils.import_utils import ( - is_causal_conv1d_available, - is_flash_attn_2_available, - is_mamba_2_ssm_available, -) +from ...utils import auto_docstring, can_return_tuple, logging +from ...utils.import_utils import is_causal_conv1d_available, is_flash_attn_2_available, is_mamba_2_ssm_available from .configuration_bamba import BambaConfig @@ -82,8 +70,6 @@ is_fast_path_available = all((selective_state_update, causal_conv1d_fn, causal_c logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "BambaConfig" - # Adapted from transformers.models.jamba.modeling_jamba.HybridMambaAttentionDynamicCache for the v2 mixer class HybridMambaAttentionDynamicCache(modeling_jamba.HybridMambaAttentionDynamicCache): @@ -782,27 +768,7 @@ class BambaDecoderLayer(JambaAttentionDecoderLayer): return outputs -BAMBA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BambaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare BambaModel outputting raw hidden-states without any specific head on top.", - BAMBA_START_DOCSTRING, -) +@auto_docstring class BambaPreTrainedModel(PreTrainedModel): config_class = BambaConfig base_model_prefix = "model" @@ -832,91 +798,8 @@ class BambaPreTrainedModel(PreTrainedModel): module.D.data.fill_(1.0) -BAMBA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`HybridMambaAttentionDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - A HybridMambaAttentionDynamicCache object containing pre-computed hidden-states (keys and values in the - self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`. - Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and - `(batch_size, d_inner, d_state)` respectively. - See the `HybridMambaAttentionDynamicCache` class for more details. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Bamba Model outputting raw hidden-states without any specific head on top.", - BAMBA_START_DOCSTRING, -) -# Adapted from transformers.models.jamba.modeling_jamba.JambaModel +@auto_docstring class BambaModel(BambaPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`BambaDecoderLayer`] - - Args: - config: BambaConfig - """ - def __init__(self, config: BambaConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -943,7 +826,7 @@ class BambaModel(BambaPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(BAMBA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1183,9 +1066,6 @@ class BambaModel(BambaPreTrainedModel): class BambaForCausalLM(LlamaForCausalLM): - @can_return_tuple - @add_start_docstrings_to_model_forward(BAMBA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1202,19 +1082,10 @@ class BambaForCausalLM(LlamaForCausalLM): **kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/bark/modeling_bark.py b/src/transformers/models/bark/modeling_bark.py index 2357fd5375..7a564393f9 100644 --- a/src/transformers/models/bark/modeling_bark.py +++ b/src/transformers/models/bark/modeling_bark.py @@ -34,8 +34,7 @@ from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, from ...modeling_outputs import CausalLMOutputWithPast, MaskedLMOutput from ...modeling_utils import PreTrainedModel, get_parameter_device from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_accelerate_available, is_torch_accelerator_available, logging, @@ -62,10 +61,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "suno/bark-small" -_CONFIG_FOR_DOC = "BarkConfig" - - class BarkSelfAttention(nn.Module): # adapted from GPTNeoSelfAttention and Bark code # BarkSelfAttention can have two attention type, i.e full attention or causal attention @@ -368,12 +363,8 @@ class BarkBlock(nn.Module): return outputs # hidden_states, ((present), attentions) +@auto_docstring class BarkPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BarkConfig supports_gradient_checkpointing = False _supports_flash_attn_2 = True @@ -418,134 +409,6 @@ class BarkPreTrainedModel(PreTrainedModel): return get_parameter_device(self) -BARK_MODEL_START_DOCSTRING = """ - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`{config}`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -BARK_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BarkConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -BARK_FINE_INPUTS_DOCSTRING = r""" - Args: - codebook_idx (`int`): - Index of the codebook that will be predicted. - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, number_of_codebooks)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. Initially, indices of the first two codebooks are obtained from the `coarse` sub-model. The rest is - predicted recursively by attending the previously predicted channels. The model predicts on windows of - length 1024. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): NOT IMPLEMENTED YET. - input_embeds (`torch.FloatTensor` of shape `(batch_size, input_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. If - `past_key_values` is used, optionally only the last `input_embeds` have to be input (see - `past_key_values`). This is useful if you want more control over how to convert `input_ids` indices into - associated vectors than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -BARK_CAUSAL_MODEL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `input_ids` of shape `(batch_size, sequence_length)`. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - input_embeds (`torch.FloatTensor` of shape `(batch_size, input_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - Here, due to `Bark` particularities, if `past_key_values` is used, `input_embeds` will be ignored and you - have to use `input_ids`. If `past_key_values` is not used and `use_cache` is set to `True`, `input_embeds` - is used in priority instead of `input_ids`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # GPT2-like autoregressive model class BarkCausalModel(BarkPreTrainedModel, GenerationMixin): config_class = BarkSubModelConfig @@ -639,7 +502,7 @@ class BarkCausalModel(BarkPreTrainedModel, GenerationMixin): "attention_mask": attention_mask, } - @add_start_docstrings_to_model_forward(BARK_CAUSAL_MODEL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -654,6 +517,13 @@ class BarkCausalModel(BarkPreTrainedModel, GenerationMixin): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithPast]: + r""" + input_embeds (`torch.FloatTensor` of shape `(batch_size, input_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + Here, due to `Bark` particularities, if `past_key_values` is used, `input_embeds` will be ignored and you + have to use `input_ids`. If `past_key_values` is not used and `use_cache` is set to `True`, `input_embeds` + is used in priority instead of `input_ids`. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -803,10 +673,11 @@ class BarkCausalModel(BarkPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """Bark semantic (or text) model. It shares the same architecture as the coarse model. - It is a GPT-2 like autoregressive model with a language modeling head on top.""", - BARK_MODEL_START_DOCSTRING.format(config="BarkSemanticConfig"), +@auto_docstring( + custom_intro=""" + Bark semantic (or text) model. It shares the same architecture as the coarse model. + It is a GPT-2 like autoregressive model with a language modeling head on top. + """ ) class BarkSemanticModel(BarkCausalModel): base_model_prefix = "semantic" @@ -912,11 +783,12 @@ class BarkSemanticModel(BarkCausalModel): return semantic_output -@add_start_docstrings( - """Bark coarse acoustics model. +@auto_docstring( + custom_intro=""" + Bark coarse acoustics model. It shares the same architecture as the semantic (or text) model. It is a GPT-2 like autoregressive model with a - language modeling head on top.""", - BARK_MODEL_START_DOCSTRING.format(config="BarkCoarseConfig"), + language modeling head on top. + """ ) class BarkCoarseModel(BarkCausalModel): base_model_prefix = "coarse_acoustics" @@ -1133,10 +1005,11 @@ class BarkCoarseModel(BarkCausalModel): return coarse_output -@add_start_docstrings( - """Bark fine acoustics model. It is a non-causal GPT-like model with `config.n_codes_total` embedding layers and - language modeling heads, one for each codebook.""", - BARK_MODEL_START_DOCSTRING.format(config="BarkFineConfig"), +@auto_docstring( + custom_intro=""" + Bark fine acoustics model. It is a non-causal GPT-like model with `config.n_codes_total` embedding layers and + language modeling heads, one for each codebook. + """ ) class BarkFineModel(BarkPreTrainedModel): base_model_prefix = "fine_acoustics" @@ -1293,7 +1166,7 @@ class BarkFineModel(BarkPreTrainedModel): if hasattr(module, "_tie_weights"): module._tie_weights() - @add_start_docstrings_to_model_forward(BARK_FINE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, codebook_idx: int, # an additional idx corresponding to the id of the codebook that will be predicted @@ -1307,6 +1180,17 @@ class BarkFineModel(BarkPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: + r""" + codebook_idx (`int`): + Index of the codebook that will be predicted. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + NOT IMPLEMENTED YET. + input_embeds (`torch.FloatTensor` of shape `(batch_size, input_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. If + `past_key_values` is used, optionally only the last `input_embeds` have to be input (see + `past_key_values`). This is useful if you want more control over how to convert `input_ids` indices into + associated vectors than the model's internal embedding lookup matrix. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1541,8 +1425,8 @@ class BarkFineModel(BarkPreTrainedModel): return fine_input -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The full Bark model, a text-to-speech model composed of 4 sub-models: - [`BarkSemanticModel`] (also referred to as the 'text' model): a causal auto-regressive transformer model that takes @@ -1557,8 +1441,7 @@ class BarkFineModel(BarkPreTrainedModel): It should be noted that each of the first three modules can support conditional speaker embeddings to condition the output sound according to specific predefined voice. - """, - BARK_START_DOCSTRING, + """ ) class BarkModel(BarkPreTrainedModel): config_class = BarkConfig diff --git a/src/transformers/models/bart/modeling_bart.py b/src/transformers/models/bart/modeling_bart.py index bc3d4dcd93..a9f9c31ac0 100755 --- a/src/transformers/models/bart/modeling_bart.py +++ b/src/transformers/models/bart/modeling_bart.py @@ -43,14 +43,7 @@ from ...modeling_outputs import ( Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_bart import BartConfig @@ -60,22 +53,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/bart-base" -_CONFIG_FOR_DOC = "BartConfig" - -# Base model docstring -_EXPECTED_OUTPUT_SHAPE = [1, 8, 768] - -# SequenceClassification docstring -_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "valhalla/bart-large-sst2" -_SEQ_CLASS_EXPECTED_LOSS = 0.0 -_SEQ_CLASS_EXPECTED_OUTPUT = "'POSITIVE'" - -# QuestionAsnwering docstring -_CHECKPOINT_FOR_QA = "valhalla/bart-large-finetuned-squadv1" -_QA_EXPECTED_LOSS = 0.59 -_QA_EXPECTED_OUTPUT = "' nice puppet'" - def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): """ @@ -739,6 +716,7 @@ class BartClassificationHead(nn.Module): return hidden_states +@auto_docstring class BartPreTrainedModel(PreTrainedModel): config_class = BartConfig base_model_prefix = "model" @@ -787,163 +765,6 @@ class BartPretrainedModel(BartPreTrainedModel): ) -BART_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BartConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BART_GENERATION_EXAMPLE = r""" - Summarization example: - - ```python - >>> from transformers import AutoTokenizer, BartForConditionalGeneration - - >>> model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") - - >>> ARTICLE_TO_SUMMARIZE = ( - ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " - ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " - ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." - ... ) - >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="pt") - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) - >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - 'PG&E scheduled the blackouts in response to forecasts for high winds amid dry conditions' - ``` - - Mask filling example: - - ```python - >>> from transformers import AutoTokenizer, BartForConditionalGeneration - - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-base") - >>> model = BartForConditionalGeneration.from_pretrained("facebook/bart-base") - - >>> TXT = "My friends are but they eat too many carbs." - >>> input_ids = tokenizer([TXT], return_tensors="pt")["input_ids"] - >>> logits = model(input_ids).logits - - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() - >>> probs = logits[0, masked_index].softmax(dim=0) - >>> values, predictions = probs.topk(5) - - >>> tokenizer.decode(predictions).split() - ['not', 'good', 'healthy', 'great', 'very'] - ``` -""" - -BART_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class BartEncoder(BartPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1419,10 +1240,7 @@ class BartDecoder(BartPreTrainedModel): ) -@add_start_docstrings( - "The bare BART Model outputting raw hidden-states without any specific head on top.", - BART_START_DOCSTRING, -) +@auto_docstring class BartModel(BartPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1465,13 +1283,7 @@ class BartModel(BartPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1490,6 +1302,35 @@ class BartModel(BartPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ # different to other models, Bart automatically creates decoder_input_ids from # input_ids if no decoder_input_ids are provided if decoder_input_ids is None and decoder_inputs_embeds is None: @@ -1560,8 +1401,10 @@ class BartModel(BartPreTrainedModel): ) -@add_start_docstrings( - "The BART Model with a language modeling head. Can be used for summarization.", BART_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The BART Model with a language modeling head. Can be used for summarization. + """ ) class BartForConditionalGeneration(BartPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1610,9 +1453,7 @@ class BartForConditionalGeneration(BartPreTrainedModel, GenerationMixin): self.model._tie_weights() self._tie_or_clone_weights(self.lm_head, self.model.shared) - @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(BART_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1633,12 +1474,78 @@ class BartForConditionalGeneration(BartPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example summarization: + + ```python + >>> from transformers import AutoTokenizer, BartForConditionalGeneration + + >>> model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn") + + >>> ARTICLE_TO_SUMMARIZE = ( + ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " + ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " + ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." + ... ) + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="pt") + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) + >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + 'PG&E scheduled the blackouts in response to forecasts for high winds amid dry conditions' + ``` + + Mask filling example: + + ```python + >>> from transformers import AutoTokenizer, BartForConditionalGeneration + + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/bart-base") + >>> model = BartForConditionalGeneration.from_pretrained("facebook/bart-base") + + >>> TXT = "My friends are but they eat too many carbs." + >>> input_ids = tokenizer([TXT], return_tensors="pt")["input_ids"] + >>> logits = model(input_ids).logits + + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = logits[0, masked_index].softmax(dim=0) + >>> values, predictions = probs.topk(5) + + >>> tokenizer.decode(predictions).split() + ['not', 'good', 'healthy', 'great', 'very'] + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1709,12 +1616,11 @@ class BartForConditionalGeneration(BartPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Bart model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - BART_START_DOCSTRING, + """ ) class BartForSequenceClassification(BartPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1732,14 +1638,7 @@ class BartForSequenceClassification(BartPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, - output_type=Seq2SeqSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1759,6 +1658,33 @@ class BartForSequenceClassification(BartPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). @@ -1839,13 +1765,7 @@ class BartForSequenceClassification(BartPreTrainedModel): ) -@add_start_docstrings( - """ - BART Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - BART_START_DOCSTRING, -) +@auto_docstring class BartForQuestionAnswering(BartPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1861,14 +1781,7 @@ class BartForQuestionAnswering(BartPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_QA, - output_type=Seq2SeqQuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_loss=_QA_EXPECTED_LOSS, - expected_output=_QA_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1889,14 +1802,33 @@ class BartForQuestionAnswering(BartPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if start_positions is not None and end_positions is not None: @@ -1978,11 +1910,10 @@ class BartDecoderWrapper(BartPreTrainedModel): return self.decoder(*args, **kwargs) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BART decoder with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - BART_START_DOCSTRING, + """ ) class BartForCausalLM(BartPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -2017,7 +1948,7 @@ class BartForCausalLM(BartPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2035,72 +1966,15 @@ class BartForCausalLM(BartPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/beit/image_processing_beit_fast.py b/src/transformers/models/beit/image_processing_beit_fast.py index e9471c62c9..5f090b9276 100644 --- a/src/transformers/models/beit/image_processing_beit_fast.py +++ b/src/transformers/models/beit/image_processing_beit_fast.py @@ -21,7 +21,6 @@ from torchvision.transforms import functional as F from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -40,24 +39,21 @@ from ...image_utils import ( validate_kwargs, ) from ...processing_utils import Unpack -from ...utils import TensorType, add_start_docstrings -from ...utils.deprecation import deprecate_kwarg +from ...utils import TensorType, auto_docstring class BeitFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): - do_reduce_labels: Optional[bool] - - -@add_start_docstrings( - "Constructs a fast Beit image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, """ do_reduce_labels (`bool`, *optional*, defaults to `self.do_reduce_labels`): Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. - """, -) + """ + + do_reduce_labels: Optional[bool] + + +@auto_docstring class BeitImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = IMAGENET_STANDARD_MEAN @@ -72,6 +68,9 @@ class BeitImageProcessorFast(BaseImageProcessorFast): do_reduce_labels = False valid_kwargs = BeitFastImageProcessorKwargs + def __init__(self, **kwargs: Unpack[BeitFastImageProcessorKwargs]): + super().__init__(**kwargs) + @classmethod def from_dict(cls, image_processor_dict: Dict[str, Any], **kwargs): """ @@ -170,23 +169,17 @@ class BeitImageProcessorFast(BaseImageProcessorFast): # be passed in as positional arguments. return super().__call__(images, segmentation_maps=segmentation_maps, **kwargs) - @deprecate_kwarg("reduce_labels", new_name="do_reduce_labels", version="4.41.0") - @add_start_docstrings( - "Constructs a fast Beit image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - do_reduce_labels (`bool`, *optional*, defaults to `self.do_reduce_labels`): - Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 - is used for background, and background itself is not included in all classes of a dataset (e.g. - ADE20k). The background label will be replaced by 255. - """, - ) + @auto_docstring def preprocess( self, images: ImageInput, segmentation_maps: Optional[ImageInput] = None, - **kwargs: Unpack[DefaultFastImageProcessorKwargs], + **kwargs: Unpack[BeitFastImageProcessorKwargs], ) -> BatchFeature: + r""" + segmentation_maps (`ImageInput`, *optional*): + The segmentation maps to preprocess. + """ validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys()) # Set default kwargs from self. This ensures that if a kwarg is not provided # by the user, it gets its default value from the instance, or is set to None. diff --git a/src/transformers/models/beit/modeling_beit.py b/src/transformers/models/beit/modeling_beit.py index 94c6f19d43..24e16ee8bf 100755 --- a/src/transformers/models/beit/modeling_beit.py +++ b/src/transformers/models/beit/modeling_beit.py @@ -36,14 +36,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import compile_compatible_method_lru_cache, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from ...utils.backbone_utils import BackboneMixin from .configuration_beit import BeitConfig @@ -51,10 +44,8 @@ from .configuration_beit import BeitConfig logger = logging.get_logger(__name__) # General docstring -_CONFIG_FOR_DOC = "BeitConfig" # Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/beit-base-patch16-224-pt22k" _EXPECTED_OUTPUT_SHAPE = [1, 197, 768] # Image classification docstring @@ -741,12 +732,8 @@ class BeitEncoder(nn.Module): ) +@auto_docstring class BeitPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BeitConfig base_model_prefix = "beit" main_input_name = "pixel_values" @@ -784,48 +771,13 @@ class BeitPreTrainedModel(PreTrainedModel): module.lambda_2.data.fill_(self.config.layer_scale_init_value) -BEIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`BeitConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BEIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`BeitImageProcessor.__call__`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Beit Model transformer outputting raw hidden-states without any specific head on top.", - BEIT_START_DOCSTRING, -) +@auto_docstring class BeitModel(BeitPreTrainedModel): def __init__(self, config: BeitConfig, add_pooling_layer: bool = True) -> None: + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -851,14 +803,7 @@ class BeitModel(BeitPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(BEIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BeitModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -933,12 +878,13 @@ class BeitPooler(nn.Module): return pooled_output -@add_start_docstrings( - """Beit Model transformer with a 'language' modeling head on top. BEiT does masked image modeling by predicting +@auto_docstring( + custom_intro=""" + Beit Model transformer with a 'language' modeling head on top. BEiT does masked image modeling by predicting visual tokens of a Vector-Quantize Variational Autoencoder (VQ-VAE), whereas other vision models like ViT and DeiT predict RGB pixel values. As a result, this class is incompatible with [`AutoModelForMaskedImageModeling`], so you - will need to use [`BeitForMaskedImageModeling`] directly if you wish to do masked image modeling with BEiT.""", - BEIT_START_DOCSTRING, + will need to use [`BeitForMaskedImageModeling`] directly if you wish to do masked image modeling with BEiT. + """ ) class BeitForMaskedImageModeling(BeitPreTrainedModel): def __init__(self, config: BeitConfig) -> None: @@ -954,8 +900,7 @@ class BeitForMaskedImageModeling(BeitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -970,14 +915,11 @@ class BeitForMaskedImageModeling(BeitPreTrainedModel): r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1035,12 +977,11 @@ class BeitForMaskedImageModeling(BeitPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final hidden states of the patch tokens) e.g. for ImageNet. - """, - BEIT_START_DOCSTRING, + """ ) class BeitForImageClassification(BeitPreTrainedModel): def __init__(self, config: BeitConfig) -> None: @@ -1055,13 +996,7 @@ class BeitForImageClassification(BeitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BEIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1361,12 +1296,7 @@ class BeitFCNHead(nn.Module): return output -@add_start_docstrings( - """ - Beit Model transformer with a semantic segmentation head on top e.g. for ADE20k, CityScapes. - """, - BEIT_START_DOCSTRING, -) +@auto_docstring class BeitForSemanticSegmentation(BeitPreTrainedModel): def __init__(self, config: BeitConfig) -> None: super().__init__(config) @@ -1419,8 +1349,7 @@ class BeitForSemanticSegmentation(BeitPreTrainedModel): return loss - @add_start_docstrings_to_model_forward(BEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1436,8 +1365,6 @@ class BeitForSemanticSegmentation(BeitPreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1514,11 +1441,10 @@ class BeitForSemanticSegmentation(BeitPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BEiT backbone, to be used with frameworks like DETR and MaskFormer. - """, - BEIT_START_DOCSTRING, + """ ) class BeitBackbone(BeitPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -1554,8 +1480,7 @@ class BeitBackbone(BeitPreTrainedModel, BackboneMixin): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(BEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -1563,9 +1488,7 @@ class BeitBackbone(BeitPreTrainedModel, BackboneMixin): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/bert/modeling_bert.py b/src/transformers/models/bert/modeling_bert.py index 3a311f4a73..dd738ea96e 100755 --- a/src/transformers/models/bert/modeling_bert.py +++ b/src/transformers/models/bert/modeling_bert.py @@ -29,10 +29,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - _prepare_4d_attention_mask_for_sdpa, - _prepare_4d_causal_attention_mask_for_sdpa, -) +from ...modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -46,42 +43,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - get_torch_version, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, get_torch_version, logging from .configuration_bert import BertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google-bert/bert-base-uncased" -_CONFIG_FOR_DOC = "BertConfig" - -# TokenClassification docstring -_CHECKPOINT_FOR_TOKEN_CLASSIFICATION = "dbmdz/bert-large-cased-finetuned-conll03-english" -_TOKEN_CLASS_EXPECTED_OUTPUT = ( - "['O', 'I-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'I-LOC', 'I-LOC'] " -) -_TOKEN_CLASS_EXPECTED_LOSS = 0.01 - -# QuestionAnswering docstring -_CHECKPOINT_FOR_QA = "deepset/bert-base-cased-squad2" -_QA_EXPECTED_OUTPUT = "'a nice puppet'" -_QA_EXPECTED_LOSS = 7.41 -_QA_TARGET_START_INDEX = 14 -_QA_TARGET_END_INDEX = 15 - -# SequenceClassification docstring -_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "textattack/bert-base-uncased-yelp-polarity" -_SEQ_CLASS_EXPECTED_OUTPUT = "'LABEL_1'" -_SEQ_CLASS_EXPECTED_LOSS = 0.01 - def load_tf_weights_in_bert(model, config, tf_checkpoint_path): """Load tf checkpoints in a pytorch model.""" @@ -821,12 +788,8 @@ class BertPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class BertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BertConfig load_tf_weights = load_tf_weights_in_bert base_model_prefix = "bert" @@ -886,79 +849,8 @@ class BertForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -BERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`or `(batch_size, sequence_length, target_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", - BERT_START_DOCSTRING, -) -class BertModel(BertPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, @@ -968,10 +860,15 @@ class BertModel(BertPreTrainedModel): to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. """ - +) +class BertModel(BertPreTrainedModel): _no_split_modules = ["BertEmbeddings", "BertLayer"] def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1000,12 +897,7 @@ class BertModel(BertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1022,26 +914,6 @@ class BertModel(BertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, target_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1169,12 +1041,11 @@ class BertModel(BertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. - """, - BERT_START_DOCSTRING, + """ ) class BertForPreTraining(BertPreTrainedModel): _tied_weights_keys = ["predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1195,8 +1066,7 @@ class BertForPreTraining(BertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1212,20 +1082,16 @@ class BertForPreTraining(BertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BertForPreTrainingOutput]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), - the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for computing the next sequence prediction (classification) loss. Input should be a sequence - pair (see `input_ids` docstring) Indices should be in `[0, 1]`: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence + pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - - 0 indicates sequence B is a continuation of sequence A, - - 1 indicates sequence B is a random sequence. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. - - Returns: + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. Example: @@ -1280,8 +1146,10 @@ class BertForPreTraining(BertPreTrainedModel): ) -@add_start_docstrings( - """Bert Model with a `language modeling` head on top for CLM fine-tuning.""", BERT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + Bert Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class BertLMHeadModel(BertPreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1305,12 +1173,7 @@ class BertLMHeadModel(BertPreTrainedModel, GenerationMixin): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1330,28 +1193,10 @@ class BertLMHeadModel(BertPreTrainedModel, GenerationMixin): **loss_kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -1402,7 +1247,7 @@ class BertLMHeadModel(BertPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings("""Bert Model with a `language modeling` head on top.""", BERT_START_DOCSTRING) +@auto_docstring class BertForMaskedLM(BertPreTrainedModel): _tied_weights_keys = ["predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1428,14 +1273,7 @@ class BertForMaskedLM(BertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'paris'", - expected_loss=0.88, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1518,9 +1356,10 @@ class BertForMaskedLM(BertPreTrainedModel): return False -@add_start_docstrings( - """Bert Model with a `next sentence prediction (classification)` head on top.""", - BERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Bert Model with a `next sentence prediction (classification)` head on top. + """ ) class BertForNextSentencePrediction(BertPreTrainedModel): def __init__(self, config): @@ -1532,8 +1371,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=NextSentencePredictorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1556,8 +1394,6 @@ class BertForNextSentencePrediction(BertPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - Returns: - Example: ```python @@ -1620,12 +1456,11 @@ class BertForNextSentencePrediction(BertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - BERT_START_DOCSTRING, + """ ) class BertForSequenceClassification(BertPreTrainedModel): def __init__(self, config): @@ -1643,14 +1478,7 @@ class BertForSequenceClassification(BertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1723,13 +1551,7 @@ class BertForSequenceClassification(BertPreTrainedModel): ) -@add_start_docstrings( - """ - Bert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - BERT_START_DOCSTRING, -) +@auto_docstring class BertForMultipleChoice(BertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1744,12 +1566,7 @@ class BertForMultipleChoice(BertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1764,6 +1581,30 @@ class BertForMultipleChoice(BertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1817,13 +1658,7 @@ class BertForMultipleChoice(BertPreTrainedModel): ) -@add_start_docstrings( - """ - Bert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - BERT_START_DOCSTRING, -) +@auto_docstring class BertForTokenClassification(BertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1839,14 +1674,7 @@ class BertForTokenClassification(BertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_TOKEN_CLASSIFICATION, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_TOKEN_CLASS_EXPECTED_OUTPUT, - expected_loss=_TOKEN_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1900,13 +1728,7 @@ class BertForTokenClassification(BertPreTrainedModel): ) -@add_start_docstrings( - """ - Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - BERT_START_DOCSTRING, -) +@auto_docstring class BertForQuestionAnswering(BertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1918,16 +1740,7 @@ class BertForQuestionAnswering(BertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_QA, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - qa_target_start_index=_QA_TARGET_START_INDEX, - qa_target_end_index=_QA_TARGET_END_INDEX, - expected_output=_QA_EXPECTED_OUTPUT, - expected_loss=_QA_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1942,16 +1755,6 @@ class BertForQuestionAnswering(BertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.bert( diff --git a/src/transformers/models/bert_generation/modeling_bert_generation.py b/src/transformers/models/bert_generation/modeling_bert_generation.py index 75fd1b1716..6dee1db6fb 100755 --- a/src/transformers/models/bert_generation/modeling_bert_generation.py +++ b/src/transformers/models/bert_generation/modeling_bert_generation.py @@ -27,20 +27,14 @@ from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, Causa from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_bert_generation import BertGenerationConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/bert_for_seq_generation_L-24_bbc_encoder" -_CONFIG_FOR_DOC = "BertGenerationConfig" - # Copied from transformers.models.bert.modeling_bert.BertSelfOutput with Bert->BertGeneration class BertGenerationSelfOutput(nn.Module): @@ -583,12 +577,8 @@ class BertGenerationEmbeddings(nn.Module): return embeddings +@auto_docstring class BertGenerationPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BertGenerationConfig base_model_prefix = "bert" supports_gradient_checkpointing = True @@ -612,67 +602,10 @@ class BertGenerationPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -BERT_GENERATION_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BertGenerationConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BERT_GENERATION_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and - [`PreTrainedTokenizer.encode`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare BertGeneration model transformer outputting raw hidden-states without any specific head on top.", - BERT_GENERATION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare BertGeneration model transformer outputting raw hidden-states without any specific head on top. + """ ) class BertGenerationEncoder(BertGenerationPreTrainedModel): """ @@ -715,12 +648,7 @@ class BertGenerationEncoder(BertGenerationPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(BERT_GENERATION_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -737,24 +665,6 @@ class BertGenerationEncoder(BertGenerationPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, # NOOP kwargs, for now ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: `1` for - tokens that are NOT MASKED, `0` for MASKED tokens. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -860,9 +770,10 @@ class BertGenerationOnlyLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """BertGeneration Model with a `language modeling` head on top for CLM fine-tuning.""", - BERT_GENERATION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + BertGeneration Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class BertGenerationDecoder(BertGenerationPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -886,8 +797,7 @@ class BertGenerationDecoder(BertGenerationPreTrainedModel, GenerationMixin): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BERT_GENERATION_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -906,30 +816,10 @@ class BertGenerationDecoder(BertGenerationPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: diff --git a/src/transformers/models/big_bird/modeling_big_bird.py b/src/transformers/models/big_bird/modeling_big_bird.py index 3b250a056e..5bae889471 100755 --- a/src/transformers/models/big_bird/modeling_big_bird.py +++ b/src/transformers/models/big_bird/modeling_big_bird.py @@ -38,22 +38,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_big_bird import BigBirdConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/bigbird-roberta-base" -_CONFIG_FOR_DOC = "BigBirdConfig" - _TRIVIA_QA_MAPPING = { "big_bird_attention": "attention/self", @@ -1743,12 +1733,8 @@ class BigBirdPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class BigBirdPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BigBirdConfig load_tf_weights = load_tf_weights_in_big_bird base_model_prefix = "bert" @@ -1773,67 +1759,6 @@ class BigBirdPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -BIG_BIRD_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`BigBirdConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BIG_BIRD_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - @dataclass class BigBirdForPreTrainingOutput(ModelOutput): """ @@ -1903,10 +1828,7 @@ class BigBirdForQuestionAnsweringModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -@add_start_docstrings( - "The bare BigBird Model transformer outputting raw hidden-states without any specific head on top.", - BIG_BIRD_START_DOCSTRING, -) +@auto_docstring class BigBirdModel(BigBirdPreTrainedModel): """ @@ -1921,6 +1843,10 @@ class BigBirdModel(BigBirdPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.attention_type = self.config.attention_type self.config = config @@ -1964,12 +1890,7 @@ class BigBirdModel(BigBirdPreTrainedModel): self.attention_type = value self.encoder.set_attention_type(value) - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1987,25 +1908,6 @@ class BigBirdModel(BigBirdPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, # NOOP kwargs, for now ) -> Union[BaseModelOutputWithPoolingAndCrossAttentions, Tuple[torch.FloatTensor]]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -2266,8 +2168,7 @@ class BigBirdForPreTraining(BigBirdPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BigBirdForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2294,10 +2195,6 @@ class BigBirdForPreTraining(BigBirdPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. - - Returns: Example: @@ -2353,7 +2250,7 @@ class BigBirdForPreTraining(BigBirdPreTrainedModel): ) -@add_start_docstrings("""BigBird Model with a `language modeling` head on top.""", BIG_BIRD_START_DOCSTRING) +@auto_docstring class BigBirdForMaskedLM(BigBirdPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -2379,8 +2276,7 @@ class BigBirdForMaskedLM(BigBirdPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2402,8 +2298,6 @@ class BigBirdForMaskedLM(BigBirdPreTrainedModel): config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: - Example: ```python @@ -2496,8 +2390,10 @@ class BigBirdForMaskedLM(BigBirdPreTrainedModel): return {"input_ids": input_ids, "attention_mask": attention_mask} -@add_start_docstrings( - """BigBird Model with a `language modeling` head on top for CLM fine-tuning.""", BIG_BIRD_START_DOCSTRING +@auto_docstring( + custom_intro=""" + BigBird Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class BigBirdForCausalLM(BigBirdPreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -2521,12 +2417,7 @@ class BigBirdForCausalLM(BigBirdPreTrainedModel, GenerationMixin): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2546,27 +2437,10 @@ class BigBirdForCausalLM(BigBirdPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[CausalLMOutputWithCrossAttentions, Tuple[torch.FloatTensor]]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2646,12 +2520,11 @@ class BigBirdClassificationHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BigBird Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - BIG_BIRD_START_DOCSTRING, + """ ) class BigBirdForSequenceClassification(BigBirdPreTrainedModel): def __init__(self, config): @@ -2664,8 +2537,7 @@ class BigBirdForSequenceClassification(BigBirdPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2685,8 +2557,6 @@ class BigBirdForSequenceClassification(BigBirdPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Example: ```python @@ -2774,13 +2644,7 @@ class BigBirdForSequenceClassification(BigBirdPreTrainedModel): ) -@add_start_docstrings( - """ - BigBird Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - BIG_BIRD_START_DOCSTRING, -) +@auto_docstring class BigBirdForMultipleChoice(BigBirdPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2792,14 +2656,7 @@ class BigBirdForMultipleChoice(BigBirdPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2814,6 +2671,30 @@ class BigBirdForMultipleChoice(BigBirdPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[MultipleChoiceModelOutput, Tuple[torch.FloatTensor]]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -2867,13 +2748,7 @@ class BigBirdForMultipleChoice(BigBirdPreTrainedModel): ) -@add_start_docstrings( - """ - BigBird Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - BIG_BIRD_START_DOCSTRING, -) +@auto_docstring class BigBirdForTokenClassification(BigBirdPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2889,12 +2764,7 @@ class BigBirdForTokenClassification(BigBirdPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2966,15 +2836,13 @@ class BigBirdForQuestionAnsweringHead(nn.Module): return hidden_states -@add_start_docstrings( - """ - BigBird Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - BIG_BIRD_START_DOCSTRING, -) +@auto_docstring class BigBirdForQuestionAnswering(BigBirdPreTrainedModel): def __init__(self, config, add_pooling_layer=False): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) config.num_labels = 2 @@ -2987,13 +2855,12 @@ class BigBirdForQuestionAnswering(BigBirdPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIG_BIRD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BigBirdForQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.FloatTensor] = None, - question_lengths: Optional[torch.Tensor] = None, + question_lengths: Optional[torch.LongTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, @@ -3005,16 +2872,8 @@ class BigBirdForQuestionAnswering(BigBirdPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[BigBirdForQuestionAnsweringModelOutput, Tuple[torch.FloatTensor]]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - - Returns: + question_lengths (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*): + The lengths of the questions in the batch. Example: diff --git a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py index ee81c6b3af..efcf2dc443 100755 --- a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py +++ b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py @@ -36,21 +36,12 @@ from ...modeling_outputs import ( Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_bigbird_pegasus import BigBirdPegasusConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/bigbird-pegasus-large-arxiv" -_CONFIG_FOR_DOC = "BigBirdPegasusConfig" _EXPECTED_OUTPUT_SHAPE = [1, 7, 1024] @@ -1564,6 +1555,7 @@ class BigBirdPegasusClassificationHead(nn.Module): return hidden_states +@auto_docstring class BigBirdPegasusPreTrainedModel(PreTrainedModel): config_class = BigBirdPegasusConfig base_model_prefix = "model" @@ -1594,149 +1586,6 @@ class BigBirdPegasusPreTrainedModel(PreTrainedModel): return dummy_inputs -BIGBIRD_PEGASUS_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BigBirdPegasusConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BIGBIRD_PEGASUS_GENERATION_EXAMPLE = r""" - Summarization example: - - ```python - >>> from transformers import AutoTokenizer, BigBirdPegasusForConditionalGeneration - - >>> model = BigBirdPegasusForConditionalGeneration.from_pretrained("google/bigbird-pegasus-large-arxiv") - >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv") - - >>> ARTICLE_TO_SUMMARIZE = ( - ... "The dominant sequence transduction models are based on complex recurrent or convolutional neural " - ... "networks in an encoder-decoder configuration. The best performing models also connect the encoder " - ... "and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, " - ... "based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. " - ... "Experiments on two machine translation tasks show these models to be superior in quality " - ... "while being more parallelizable and requiring significantly less time to train." - ... ) - >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=4096, return_tensors="pt", truncation=True) - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=15) - >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - 'dominant sequence models are based on recurrent or convolutional neural networks .' - ``` -""" - -BIGBIRD_PEGASUS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Provide for translation and summarization training. By default, the model will create this tensor by - shifting the `input_ids` to the right, following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read - [`modeling_bigbird_pegasus._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in - [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - - decoder_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -BIGBIRD_PEGASUS_STANDALONE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class BigBirdPegasusEncoder(BigBirdPegasusPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -2293,10 +2142,7 @@ class BigBirdPegasusDecoder(BigBirdPegasusPreTrainedModel): ) -@add_start_docstrings( - "The bare BigBirdPegasus Model outputting raw hidden-states without any specific head on top.", - BIGBIRD_PEGASUS_START_DOCSTRING, -) +@auto_docstring class BigBirdPegasusModel(BigBirdPegasusPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -2334,14 +2180,7 @@ class BigBirdPegasusModel(BigBirdPegasusPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(BIGBIRD_PEGASUS_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) - # Copied from transformers.models.bart.modeling_bart.BartModel.forward with Bart->BigBirdPegasus + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2360,6 +2199,23 @@ class BigBirdPegasusModel(BigBirdPegasusPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for translation and summarization training. By default, the model will create this tensor by + shifting the `input_ids` to the right, following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_bigbird_pegasus._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in + [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + decoder_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ # different to other models, BigBirdPegasus automatically creates decoder_input_ids from # input_ids if no decoder_input_ids are provided if decoder_input_ids is None and decoder_inputs_embeds is None: @@ -2430,9 +2286,10 @@ class BigBirdPegasusModel(BigBirdPegasusPreTrainedModel): ) -@add_start_docstrings( - "The BigBirdPegasus Model with a language modeling head. Can be used for summarization.", - BIGBIRD_PEGASUS_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The BigBirdPegasus Model with a language modeling head. Can be used for summarization. + """ ) # Copied from transformers.models.bart.modeling_bart.BartForConditionalGeneration with Bart->BigBirdPegasus, BART->BIGBIRD_PEGASUS class BigBirdPegasusForConditionalGeneration(BigBirdPegasusPreTrainedModel, GenerationMixin): @@ -2482,9 +2339,8 @@ class BigBirdPegasusForConditionalGeneration(BigBirdPegasusPreTrainedModel, Gene self.model._tie_weights() self._tie_or_clone_weights(self.lm_head, self.model.shared) - @add_start_docstrings_to_model_forward(BIGBIRD_PEGASUS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(BIGBIRD_PEGASUS_GENERATION_EXAMPLE) + @auto_docstring + # Ignore copy def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2505,12 +2361,49 @@ class BigBirdPegasusForConditionalGeneration(BigBirdPegasusPreTrainedModel, Gene return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for translation and summarization training. By default, the model will create this tensor by + shifting the `input_ids` to the right, following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_bigbird_pegasus._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in + [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + decoder_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example summarization: + + ```python + >>> from transformers import AutoTokenizer, BigBirdPegasusForConditionalGeneration + + >>> model = BigBirdPegasusForConditionalGeneration.from_pretrained("google/bigbird-pegasus-large-arxiv") + >>> tokenizer = AutoTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv") + + >>> ARTICLE_TO_SUMMARIZE = ( + ... "The dominant sequence transduction models are based on complex recurrent or convolutional neural " + ... "networks in an encoder-decoder configuration. The best performing models also connect the encoder " + ... "and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, " + ... "based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. " + ... "Experiments on two machine translation tasks show these models to be superior in quality " + ... "while being more parallelizable and requiring significantly less time to train." + ... ) + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=4096, return_tensors="pt", truncation=True) + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=15) + >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + 'dominant sequence models are based on recurrent or convolutional neural networks .' + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2581,12 +2474,11 @@ class BigBirdPegasusForConditionalGeneration(BigBirdPegasusPreTrainedModel, Gene return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BigBirdPegasus model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - BIGBIRD_PEGASUS_START_DOCSTRING, + """ ) class BigBirdPegasusForSequenceClassification(BigBirdPegasusPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -2604,13 +2496,7 @@ class BigBirdPegasusForSequenceClassification(BigBirdPegasusPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIGBIRD_PEGASUS_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) - # Copied from transformers.models.bart.modeling_bart.BartForSequenceClassification.forward + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2630,6 +2516,21 @@ class BigBirdPegasusForSequenceClassification(BigBirdPegasusPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for translation and summarization training. By default, the model will create this tensor by + shifting the `input_ids` to the right, following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_bigbird_pegasus._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in + [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + decoder_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). @@ -2710,13 +2611,7 @@ class BigBirdPegasusForSequenceClassification(BigBirdPegasusPreTrainedModel): ) -@add_start_docstrings( - """ - BigBirdPegasus Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - BIGBIRD_PEGASUS_START_DOCSTRING, -) +@auto_docstring class BigBirdPegasusForQuestionAnswering(BigBirdPegasusPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -2732,13 +2627,7 @@ class BigBirdPegasusForQuestionAnswering(BigBirdPegasusPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIGBIRD_PEGASUS_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqQuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) - # Copied from transformers.models.bart.modeling_bart.BartForQuestionAnswering.forward + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2759,14 +2648,21 @@ class BigBirdPegasusForQuestionAnswering(BigBirdPegasusPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for translation and summarization training. By default, the model will create this tensor by + shifting the `input_ids` to the right, following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_bigbird_pegasus._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in + [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + decoder_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if start_positions is not None and end_positions is not None: @@ -2882,7 +2778,7 @@ class BigBirdPegasusForCausalLM(BigBirdPegasusPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2900,72 +2796,15 @@ class BigBirdPegasusForCausalLM(BigBirdPegasusPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/biogpt/modeling_biogpt.py b/src/transformers/models/biogpt/modeling_biogpt.py index 51f298098b..7fdb5a5618 100755 --- a/src/transformers/models/biogpt/modeling_biogpt.py +++ b/src/transformers/models/biogpt/modeling_biogpt.py @@ -33,9 +33,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) from .configuration_biogpt import BioGptConfig @@ -43,9 +41,6 @@ from .configuration_biogpt import BioGptConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/biogpt" -_CONFIG_FOR_DOC = "BioGptConfig" - # copied from transformers.models.opt.modeling_opt.OPTLearnedPositionalEmbedding with OPT->BioGpt # TODO @ArthurZucker bring copied from back @@ -444,12 +439,8 @@ class BioGptDecoderLayer(nn.Module): return outputs +@auto_docstring class BioGptPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BioGptConfig base_model_prefix = "biogpt" supports_gradient_checkpointing = True @@ -472,76 +463,7 @@ class BioGptPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -BIOGPT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~BioGptConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BIOGPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare BioGPT Model transformer outputting raw hidden-states without any specific head on top.", - BIOGPT_START_DOCSTRING, -) +@auto_docstring class BioGptModel(BioGptPreTrainedModel): def __init__(self, config: BioGptConfig): super().__init__(config) @@ -571,12 +493,7 @@ class BioGptModel(BioGptPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -719,8 +636,10 @@ class BioGptModel(BioGptPreTrainedModel): ) -@add_start_docstrings( - """BioGPT Model with a `language modeling` head on top for CLM fine-tuning.""", BIOGPT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + BioGPT Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class BioGptForCausalLM(BioGptPreTrainedModel, GenerationMixin): _tied_weights_keys = ["output_projection.weight"] @@ -740,12 +659,7 @@ class BioGptForCausalLM(BioGptPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.output_projection = new_embeddings - @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -815,13 +729,7 @@ class BioGptForCausalLM(BioGptPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ - BioGPT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - BIOGPT_START_DOCSTRING, -) +@auto_docstring class BioGptForTokenClassification(BioGptPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -837,12 +745,7 @@ class BioGptForTokenClassification(BioGptPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -907,8 +810,8 @@ class BioGptForTokenClassification(BioGptPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The BioGpt Model transformer with a sequence classification head on top (linear layer). [`BioGptForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -919,8 +822,7 @@ class BioGptForTokenClassification(BioGptPreTrainedModel): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - BIOGPT_START_DOCSTRING, + """ ) class BioGptForSequenceClassification(BioGptPreTrainedModel): def __init__(self, config: BioGptConfig): @@ -932,12 +834,7 @@ class BioGptForSequenceClassification(BioGptPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/bit/image_processing_bit_fast.py b/src/transformers/models/bit/image_processing_bit_fast.py index 19b3fbfd32..46c0e94f3d 100644 --- a/src/transformers/models/bit/image_processing_bit_fast.py +++ b/src/transformers/models/bit/image_processing_bit_fast.py @@ -14,18 +14,12 @@ # limitations under the License. """Fast Image processor class for BiT.""" -from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BaseImageProcessorFast, -) +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast Bit image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class BitImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN diff --git a/src/transformers/models/bit/modeling_bit.py b/src/transformers/models/bit/modeling_bit.py index d71e3bf946..f1a6ee34e7 100644 --- a/src/transformers/models/bit/modeling_bit.py +++ b/src/transformers/models/bit/modeling_bit.py @@ -32,30 +32,13 @@ from ...modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_bit import BitConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "BitConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "google/bit-50" -_EXPECTED_OUTPUT_SHAPE = [1, 2048, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "google/bit-50" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tiger cat" - def get_padding_value(padding=None, kernel_size=7, stride=1, dilation=1) -> Tuple[Tuple, bool]: r""" @@ -646,12 +629,8 @@ class BitEncoder(nn.Module): ) +@auto_docstring class BitPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BitConfig base_model_prefix = "bit" main_input_name = "pixel_values" @@ -672,35 +651,7 @@ class BitPreTrainedModel(PreTrainedModel): nn.init.constant_(module.bias, 0) -BIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`BitConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`BitImageProcessor.__call__`] - for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare BiT model outputting raw features without any specific head on top.", - BIT_START_DOCSTRING, -) +@auto_docstring class BitModel(BitPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -719,14 +670,7 @@ class BitModel(BitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BaseModelOutputWithPoolingAndNoAttention: @@ -757,12 +701,11 @@ class BitModel(BitPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BiT Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - BIT_START_DOCSTRING, + """ ) class BitForImageClassification(BitPreTrainedModel): def __init__(self, config): @@ -777,13 +720,7 @@ class BitForImageClassification(BitPreTrainedModel): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -834,11 +771,10 @@ class BitForImageClassification(BitPreTrainedModel): return ImageClassifierOutputWithNoAttention(loss=loss, logits=logits, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BiT backbone, to be used with frameworks like DETR and MaskFormer. - """, - BIT_START_DOCSTRING, + """ ) class BitBackbone(BitPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -851,14 +787,11 @@ class BitBackbone(BitPreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/bitnet/modeling_bitnet.py b/src/transformers/models/bitnet/modeling_bitnet.py index 64d1f24f8b..48076bfb78 100644 --- a/src/transformers/models/bitnet/modeling_bitnet.py +++ b/src/transformers/models/bitnet/modeling_bitnet.py @@ -34,15 +34,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_bitnet import BitNetConfig @@ -53,7 +45,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "BitNetConfig" @use_kernel_forward_from_hub("RMSNorm") @@ -332,27 +323,7 @@ class BitNetRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -BITNET_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BitNetConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare BitNet Model outputting raw hidden-states without any specific head on top.", - BITNET_START_DOCSTRING, -) +@auto_docstring class BitNetPreTrainedModel(PreTrainedModel): config_class = BitNetConfig base_model_prefix = "model" @@ -381,88 +352,8 @@ class BitNetPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -BITNET_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare BitNet Model outputting raw hidden-states without any specific head on top.", - BITNET_START_DOCSTRING, -) +@auto_docstring class BitNetModel(BitNetPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`BitNetDecoderLayer`] - - Args: - config: BitNetConfig - """ - def __init__(self, config: BitNetConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -486,7 +377,7 @@ class BitNetModel(BitNetPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(BITNET_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -708,6 +599,7 @@ class BitNetModel(BitNetPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class BitNetForCausalLM(BitNetPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = None @@ -741,8 +633,7 @@ class BitNetForCausalLM(BitNetPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(BITNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -759,13 +650,10 @@ class BitNetForCausalLM(BitNetPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`. Example: diff --git a/src/transformers/models/bitnet/modular_bitnet.py b/src/transformers/models/bitnet/modular_bitnet.py index 824ac6f0f3..0c0d133cb5 100644 --- a/src/transformers/models/bitnet/modular_bitnet.py +++ b/src/transformers/models/bitnet/modular_bitnet.py @@ -22,9 +22,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import ( - logging, -) +from ...utils import logging from ..gemma.modeling_gemma import GemmaMLP from ..llama.modeling_llama import ( LlamaAttention, @@ -40,8 +38,6 @@ from .configuration_bitnet import BitNetConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/bitnet-b1.58-2B-4T" - class BitNetRMSNorm(LlamaRMSNorm): pass @@ -132,13 +128,10 @@ class BitNetForCausalLM(LlamaForCausalLM): **super_kwargs, ) -> CausalLMOutputWithPast: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`. Example: diff --git a/src/transformers/models/blenderbot/modeling_blenderbot.py b/src/transformers/models/blenderbot/modeling_blenderbot.py index 9ce51bebe7..1eef927642 100755 --- a/src/transformers/models/blenderbot/modeling_blenderbot.py +++ b/src/transformers/models/blenderbot/modeling_blenderbot.py @@ -36,22 +36,13 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..blenderbot_small import BlenderbotSmallForConditionalGeneration, BlenderbotSmallModel from .configuration_blenderbot import BlenderbotConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "BlenderbotConfig" -_CHECKPOINT_FOR_DOC = "facebook/blenderbot-400M-distill" - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -453,6 +444,7 @@ class BlenderbotDecoderLayer(nn.Module): return outputs +@auto_docstring class BlenderbotPreTrainedModel(PreTrainedModel): config_class = BlenderbotConfig base_model_prefix = "model" @@ -481,147 +473,6 @@ class BlenderbotPreTrainedModel(PreTrainedModel): return dummy_inputs -BLENDERBOT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BlenderbotConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BLENDERBOT_GENERATION_EXAMPLE = r""" - Conversation example: - - ```python - >>> from transformers import AutoTokenizer, BlenderbotForConditionalGeneration - - >>> mname = "facebook/blenderbot-400M-distill" - >>> model = BlenderbotForConditionalGeneration.from_pretrained(mname) - >>> tokenizer = AutoTokenizer.from_pretrained(mname) - >>> UTTERANCE = "My friends are cool but they eat too many carbs." - >>> print("Human: ", UTTERANCE) - Human: My friends are cool but they eat too many carbs. - - >>> inputs = tokenizer([UTTERANCE], return_tensors="pt") - >>> reply_ids = model.generate(**inputs) - >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) - Bot: That's unfortunate. Are they trying to lose weight or are they just trying to be healthier? - - >>> REPLY = "I'm not sure" - >>> print("Human: ", REPLY) - Human: I'm not sure - - >>> NEXT_UTTERANCE = ( - ... "My friends are cool but they eat too many carbs. That's unfortunate. " - ... "Are they trying to lose weight or are they just trying to be healthier? " - ... " I'm not sure." - ... ) - >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="pt") - >>> next_reply_ids = model.generate(**inputs) - >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) - Bot: I see. Well, it's good that they're trying to change their eating habits. - ``` -""" - -BLENDERBOT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Blenderbot uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class BlenderbotEncoder(BlenderbotPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1051,10 +902,7 @@ class BlenderbotDecoder(BlenderbotPreTrainedModel): ) -@add_start_docstrings( - "The bare Blenderbot Model outputting raw hidden-states without any specific head on top.", - BLENDERBOT_START_DOCSTRING, -) +@auto_docstring class BlenderbotModel(BlenderbotPreTrainedModel): _tied_weights_keys = ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"] @@ -1097,8 +945,7 @@ class BlenderbotModel(BlenderbotPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(BLENDERBOT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1118,7 +965,26 @@ class BlenderbotModel(BlenderbotPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Blenderbot uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1192,8 +1058,10 @@ class BlenderbotModel(BlenderbotPreTrainedModel): ) -@add_start_docstrings( - "The Blenderbot Model with a language modeling head. Can be used for summarization.", BLENDERBOT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The Blenderbot Model with a language modeling head. Can be used for summarization. + """ ) class BlenderbotForConditionalGeneration(BlenderbotPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1252,9 +1120,7 @@ class BlenderbotForConditionalGeneration(BlenderbotPreTrainedModel, GenerationMi def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(BLENDERBOT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(BLENDERBOT_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1275,12 +1141,62 @@ class BlenderbotForConditionalGeneration(BlenderbotPreTrainedModel, GenerationMi return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Blenderbot uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example conversation: + + ```python + >>> from transformers import AutoTokenizer, BlenderbotForConditionalGeneration + + >>> mname = "facebook/blenderbot-400M-distill" + >>> model = BlenderbotForConditionalGeneration.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) + >>> UTTERANCE = "My friends are cool but they eat too many carbs." + >>> print("Human: ", UTTERANCE) + Human: My friends are cool but they eat too many carbs. + + >>> inputs = tokenizer([UTTERANCE], return_tensors="pt") + >>> reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) + Bot: That's unfortunate. Are they trying to lose weight or are they just trying to be healthier? + + >>> REPLY = "I'm not sure" + >>> print("Human: ", REPLY) + Human: I'm not sure + + >>> NEXT_UTTERANCE = ( + ... "My friends are cool but they eat too many carbs. That's unfortunate. " + ... "Are they trying to lose weight or are they just trying to be healthier? " + ... " I'm not sure." + ... ) + >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="pt") + >>> next_reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) + Bot: I see. Well, it's good that they're trying to change their eating habits. + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1394,7 +1310,7 @@ class BlenderbotForCausalLM(BlenderbotPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1412,72 +1328,15 @@ class BlenderbotForCausalLM(BlenderbotPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py index 7bb4a49bb8..eefb467e60 100755 --- a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py +++ b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py @@ -34,20 +34,12 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_blenderbot_small import BlenderbotSmallConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "BlenderbotSmallConfig" - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -441,6 +433,7 @@ class BlenderbotSmallDecoderLayer(nn.Module): return outputs +@auto_docstring class BlenderbotSmallPreTrainedModel(PreTrainedModel): config_class = BlenderbotSmallConfig base_model_prefix = "model" @@ -469,147 +462,6 @@ class BlenderbotSmallPreTrainedModel(PreTrainedModel): return dummy_inputs -BLENDERBOT_SMALL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BlenderbotSmallConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BLENDERBOT_SMALL_GENERATION_EXAMPLE = r""" - Conversation example: - - ```python - >>> from transformers import AutoTokenizer, BlenderbotSmallForConditionalGeneration - - >>> mname = "facebook/blenderbot_small-90M" - >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) - >>> tokenizer = AutoTokenizer.from_pretrained(mname) - >>> UTTERANCE = "My friends are cool but they eat too many carbs." - >>> print("Human: ", UTTERANCE) - Human: My friends are cool but they eat too many carbs. - - >>> inputs = tokenizer([UTTERANCE], return_tensors="pt") - >>> reply_ids = model.generate(**inputs) - >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) - Bot: what kind of carbs do they eat? i don't know much about carbs. - - >>> REPLY = "I'm not sure" - >>> print("Human: ", REPLY) - Human: I'm not sure - - >>> NEXT_UTTERANCE = ( - ... "My friends are cool but they eat too many carbs.__end__ __start__what kind of carbs do they eat? " - ... "i don't know much about carbs__end__ " - ... "__start__ I'm not sure." - ... ) - >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="pt") - >>> next_reply_ids = model.generate(**inputs) - >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) - Bot: they eat a lot of carbs. carbs are high in fat, protein, and fats. - ``` -""" - -BLENDERBOT_SMALL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class BlenderbotSmallEncoder(BlenderbotSmallPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1032,10 +884,7 @@ class BlenderbotSmallDecoder(BlenderbotSmallPreTrainedModel): ) -@add_start_docstrings( - "The bare BlenderbotSmall Model outputting raw hidden-states without any specific head on top.", - BLENDERBOT_SMALL_START_DOCSTRING, -) +@auto_docstring class BlenderbotSmallModel(BlenderbotSmallPreTrainedModel): _tied_weights_keys = ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"] @@ -1065,8 +914,7 @@ class BlenderbotSmallModel(BlenderbotSmallPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(BLENDERBOT_SMALL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1086,7 +934,26 @@ class BlenderbotSmallModel(BlenderbotSmallPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1160,9 +1027,10 @@ class BlenderbotSmallModel(BlenderbotSmallPreTrainedModel): ) -@add_start_docstrings( - "The BlenderbotSmall Model with a language modeling head. Can be used for summarization.", - BLENDERBOT_SMALL_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The BlenderbotSmall Model with a language modeling head. Can be used for summarization. + """ ) class BlenderbotSmallForConditionalGeneration(BlenderbotSmallPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1206,9 +1074,7 @@ class BlenderbotSmallForConditionalGeneration(BlenderbotSmallPreTrainedModel, Ge def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(BLENDERBOT_SMALL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(BLENDERBOT_SMALL_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1229,12 +1095,62 @@ class BlenderbotSmallForConditionalGeneration(BlenderbotSmallPreTrainedModel, Ge return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Conversation: + + ```python + >>> from transformers import AutoTokenizer, BlenderbotSmallForConditionalGeneration + + >>> mname = "facebook/blenderbot_small-90M" + >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) + >>> UTTERANCE = "My friends are cool but they eat too many carbs." + >>> print("Human: ", UTTERANCE) + Human: My friends are cool but they eat too many carbs. + + >>> inputs = tokenizer([UTTERANCE], return_tensors="pt") + >>> reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) + Bot: what kind of carbs do they eat? i don't know much about carbs. + + >>> REPLY = "I'm not sure" + >>> print("Human: ", REPLY) + Human: I'm not sure + + >>> NEXT_UTTERANCE = ( + ... "My friends are cool but they eat too many carbs.__end__ __start__what kind of carbs do they eat? " + ... "i don't know much about carbs__end__ " + ... "__start__ I'm not sure." + ... ) + >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="pt") + >>> next_reply_ids = model.generate(**inputs) + >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) + Bot: they eat a lot of carbs. carbs are high in fat, protein, and fats. + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1348,7 +1264,7 @@ class BlenderbotSmallForCausalLM(BlenderbotSmallPreTrainedModel, GenerationMixin def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1366,72 +1282,15 @@ class BlenderbotSmallForCausalLM(BlenderbotSmallPreTrainedModel, GenerationMixin return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/blip/image_processing_blip_fast.py b/src/transformers/models/blip/image_processing_blip_fast.py index acd5bae891..2b1d14f139 100644 --- a/src/transformers/models/blip/image_processing_blip_fast.py +++ b/src/transformers/models/blip/image_processing_blip_fast.py @@ -14,15 +14,12 @@ # limitations under the License. """Fast Image processor class for BLIP.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast BLIP image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class BlipImageProcessorFast(BaseImageProcessorFast): # To be checked against the slow image processor # None values left after checking can be removed diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 3b4d44afcd..548a362ebf 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -27,22 +27,13 @@ from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_blip import BlipConfig, BlipTextConfig, BlipVisionConfig from .modeling_blip_text import BlipTextLMHeadModel, BlipTextModel logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Salesforce/blip-vqa-base" - # Copied from transformers.models.clip.modeling_clip.contrastive_loss def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: @@ -462,12 +453,8 @@ class BlipEncoderLayer(nn.Module): return outputs +@auto_docstring class BlipPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True @@ -504,108 +491,6 @@ class BlipPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -BLIP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BlipConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BLIP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoProcessor`]. See [`BlipProcessor.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -BLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`BlipImageProcessor`]. See [`BlipImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - -BLIP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoProcessor`]. See [`BlipProcessor.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`BlipImageProcessor`]. See [`BlipImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - - class BlipEncoder(nn.Module): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a @@ -707,8 +592,7 @@ class BlipVisionModel(BlipPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -717,10 +601,6 @@ class BlipVisionModel(BlipPreTrainedModel): return_dict: Optional[bool] = None, interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -759,11 +639,10 @@ class BlipVisionModel(BlipPreTrainedModel): return self.embeddings -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" This model is going to be deprecated in future versions. Please use `BlipForConditionalGeneration`, `BlipForQuestionAnswering` or `BlipForImageTextRetrieval` depending on your usecase. - """, - BLIP_START_DOCSTRING, + """ ) class BlipModel(BlipPreTrainedModel): config_class = BlipConfig @@ -810,7 +689,7 @@ class BlipModel(BlipPreTrainedModel): def set_input_embeddings(self, value): self.text_model.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -848,7 +727,7 @@ class BlipModel(BlipPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -890,7 +769,7 @@ class BlipModel(BlipPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(BLIP_INPUTS_DOCSTRING) + @auto_docstring def get_multimodal_features( self, input_ids: Optional[torch.LongTensor] = None, @@ -945,8 +824,7 @@ class BlipModel(BlipPreTrainedModel): return multimodal_features - @add_start_docstrings_to_model_forward(BLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipOutput, config_class=BlipConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -960,7 +838,8 @@ class BlipModel(BlipPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BlipOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: @@ -1042,14 +921,13 @@ class BlipModel(BlipPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP Model for image captioning. The model consists of a vision encoder and a text decoder. One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the text decoder continue the prompt. Otherwise, the decoder starts generating text from the [BOS] (beginning-of-sequence) token. will start generating the caption from the text input. If no text input is provided, the decoder will start with the [BOS] token only. - """, - BLIP_START_DOCSTRING, + """ ) class BlipForConditionalGeneration(BlipPreTrainedModel, GenerationMixin): config_class = BlipConfig @@ -1075,8 +953,7 @@ class BlipForConditionalGeneration(BlipPreTrainedModel, GenerationMixin): def set_input_embeddings(self, value): self.text_decoder.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipForConditionalGenerationModelOutput, config_class=BlipVisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1089,8 +966,6 @@ class BlipForConditionalGeneration(BlipPreTrainedModel, GenerationMixin): interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BlipForConditionalGenerationModelOutput]: r""" - Returns: - Examples: ```python @@ -1225,13 +1100,12 @@ class BlipForConditionalGeneration(BlipPreTrainedModel, GenerationMixin): return outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP Model for visual question answering. The model consists of a vision encoder, a text encoder as well as a text decoder. The vision encoder will encode the input image, the text encoder will encode the input question together with the encoding of the image, and the text decoder will output the answer to the question. - """, - BLIP_START_DOCSTRING, + """ ) class BlipForQuestionAnswering(BlipPreTrainedModel, GenerationMixin): config_class = BlipConfig @@ -1259,8 +1133,7 @@ class BlipForQuestionAnswering(BlipPreTrainedModel, GenerationMixin): # This will return shared embeddings if they are shared else specific to encoder. return self.text_encoder.get_input_embeddings() - @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) + @auto_docstring def forward( self, input_ids: torch.LongTensor, @@ -1275,8 +1148,6 @@ class BlipForQuestionAnswering(BlipPreTrainedModel, GenerationMixin): interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BlipTextVisionModelOutput]: r""" - Returns: - Examples: ```python @@ -1459,13 +1330,12 @@ class BlipForQuestionAnswering(BlipPreTrainedModel, GenerationMixin): return outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP Model with a vision and text projector, and a classification head on top. The model is used in the context of image-text retrieval. Given an image and a text, the model returns the probability of the text being relevant to the image. - """, - BLIP_START_DOCSTRING, + """ ) class BlipForImageTextRetrieval(BlipPreTrainedModel): config_class = BlipConfig @@ -1506,8 +1376,7 @@ class BlipForImageTextRetrieval(BlipPreTrainedModel): def set_input_embeddings(self, value): self.text_encoder.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) + @auto_docstring def forward( self, input_ids: torch.LongTensor, @@ -1520,7 +1389,8 @@ class BlipForImageTextRetrieval(BlipPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BlipTextVisionModelOutput]: r""" - Returns: + use_itm_head (`bool`, *optional*, defaults to `True`): + Whether or not to use the image-text matching head. Examples: diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index fcc7b2b497..ff5ebb952a 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -35,23 +35,13 @@ from ...modeling_outputs import ( from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - LossKwargs, - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import LossKwargs, ModelOutput, auto_docstring, logging, torch_int from ..auto import AutoModelForCausalLM, AutoModelForSeq2SeqLM from .configuration_blip_2 import Blip2Config, Blip2QFormerConfig, Blip2VisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Salesforce/blip2-opt-2.7b" - @dataclass class Blip2ForConditionalGenerationModelOutput(ModelOutput): @@ -431,12 +421,8 @@ class Blip2EncoderLayer(nn.Module): return outputs +@auto_docstring class Blip2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Blip2Config base_model_prefix = "blip" supports_gradient_checkpointing = True @@ -483,255 +469,6 @@ class Blip2PreTrainedModel(PreTrainedModel): module.query_tokens.data.zero_() -BLIP_2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Blip2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BLIP_2_QFORMER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Blip2QFormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BLIP_2_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for - details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - -BLIP_2_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 - Training](./t5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -BLIP_2_TEXT_WITH_PROJECTION_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -BLIP_2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for - details. - - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be - provided to serve as text prompt, which the language model can continue. - - Indices can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an - encoder-decoder language model (like T5) is used. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) - - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - Only relevant in case an encoder-decoder language model (like T5) is used. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). -""" - -BLIP2_IMAGE_TEXT_RETRIEVAL_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for - details. - - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be - provided to serve as text prompt, which the language model can continue. - - Indices can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - use_image_text_matching_head (`bool`, *optional*): - Whether to return the Image-Text Matching or Contrastive scores. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -BLIP2_QFORMER_INPUTS_DOCSTRING = r""" - Args: - query_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Hidden states to be used in the attention computation. If cross-attention, - will be used for the query (i.e., key and value will use the encoder_hidden_states). - - query_length (`int`, *optional*): - Length of the query, usually based on the number of query tokens. - If no value is provided, query_length will be inferred by the query_embeds. - - attention_mask (`torch.FloatTensor`, *optional*): - Attention mask of size `(batch, sequence_length)` where padding elements - are indicated by 0. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, `optional`): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of: - shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and - value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are - used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key - value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape - `(batch_size, sequence_length)`. - - use_cache (`bool`, `optional`): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.blip.modeling_blip.BlipEncoder with Blip->Blip2 class Blip2Encoder(nn.Module): """ @@ -819,6 +556,7 @@ class Blip2Encoder(nn.Module): ) +@auto_docstring # Copied from transformers.models.blip.modeling_blip.BlipVisionModel with Blip->Blip2, BLIP->BLIP_2 class Blip2VisionModel(Blip2PreTrainedModel): main_input_name = "pixel_values" @@ -835,8 +573,7 @@ class Blip2VisionModel(Blip2PreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(BLIP_2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Blip2VisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -845,10 +582,6 @@ class Blip2VisionModel(Blip2PreTrainedModel): return_dict: Optional[bool] = None, interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1359,11 +1092,10 @@ class Blip2TextEmbeddings(nn.Module): return embeddings -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP-2 Querying Transformer (Q-Former). - """, - BLIP_2_QFORMER_START_DOCSTRING, + """ ) class Blip2QFormerModel(Blip2PreTrainedModel): _supports_attention_backend = False # adds position on attn weights before last matmul @@ -1441,10 +1173,7 @@ class Blip2QFormerModel(Blip2PreTrainedModel): extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 return extended_attention_mask - @add_start_docstrings_to_model_forward(BLIP2_QFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=Blip2QFormerConfig - ) + @auto_docstring def forward( self, query_embeds: torch.FloatTensor, @@ -1460,7 +1189,12 @@ class Blip2QFormerModel(Blip2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - Returns: + query_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Hidden states to be used in the attention computation. If cross-attention, + will be used for the query (i.e., key and value will use the encoder_hidden_states). + query_length (`int`, *optional*): + Length of the query, usually based on the number of query tokens. + If no value is provided, query_length will be inferred by the query_embeds. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1553,12 +1287,11 @@ class Blip2QFormerModel(Blip2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP-2 Model for generating text and image features. The model consists of a vision encoder, Querying Transformer (Q-Former) and a language model. - """, - BLIP_2_START_DOCSTRING, + """ ) class Blip2Model(Blip2PreTrainedModel): config_class = Blip2Config @@ -1611,7 +1344,7 @@ class Blip2Model(Blip2PreTrainedModel): self.language_model.encoder.embed_tokens = self.language_model.shared self.language_model.decoder.embed_tokens = self.language_model.shared - @add_start_docstrings_to_model_forward(BLIP_2_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1625,6 +1358,23 @@ class Blip2Model(Blip2PreTrainedModel): **kwargs: Unpack[KwargsForCausalLM], ): r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + Returns: text_outputs (`CausalLMOutputWithPast`, or `tuple(torch.FloatTensor)` if `return_dict=False`): The language model outputs. If `return_dict=True`, the output is a [`CausalLMOutputWithPast`] that @@ -1673,7 +1423,7 @@ class Blip2Model(Blip2PreTrainedModel): return text_outputs - @add_start_docstrings_to_model_forward(BLIP_2_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1719,7 +1469,7 @@ class Blip2Model(Blip2PreTrainedModel): return vision_outputs - @add_start_docstrings_to_model_forward(BLIP_2_INPUTS_DOCSTRING) + @auto_docstring def get_qformer_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1780,8 +1530,7 @@ class Blip2Model(Blip2PreTrainedModel): return query_outputs - @add_start_docstrings_to_model_forward(BLIP_2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Blip2ForConditionalGenerationModelOutput, config_class=Blip2VisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1797,7 +1546,18 @@ class Blip2Model(Blip2PreTrainedModel): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Blip2ForConditionalGenerationModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. Examples: @@ -1917,12 +1677,7 @@ class Blip2Model(Blip2PreTrainedModel): ) -@add_start_docstrings( - """ - BLIP-2 Text Model with a projection layer on top (a linear layer on top of the pooled output). - """, - BLIP_2_START_DOCSTRING, -) +@auto_docstring class Blip2TextModelWithProjection(Blip2PreTrainedModel): supports_gradient_checkpointing = False _keep_in_fp32_modules = ["query_tokens", "qformer"] @@ -1946,8 +1701,7 @@ class Blip2TextModelWithProjection(Blip2PreTrainedModel): def set_input_embeddings(self, value): self.embeddings.word_embeddings = value - @add_start_docstrings_to_model_forward(BLIP_2_TEXT_WITH_PROJECTION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Blip2TextModelOutput, config_class=Blip2Config) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1958,8 +1712,6 @@ class Blip2TextModelWithProjection(Blip2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Blip2TextModelOutput]: r""" - Returns: - Examples: ```python @@ -2016,12 +1768,7 @@ class Blip2TextModelWithProjection(Blip2PreTrainedModel): ) -@add_start_docstrings( - """ - BLIP-2 Vision Model with a projection layer on top (a linear layer on top of the pooled output). - """, - BLIP_2_START_DOCSTRING, -) +@auto_docstring class Blip2VisionModelWithProjection(Blip2PreTrainedModel): main_input_name = "pixel_values" _keep_in_fp32_modules = ["query_tokens", "qformer"] @@ -2043,8 +1790,7 @@ class Blip2VisionModelWithProjection(Blip2PreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(BLIP_2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Blip2VisionModelOutput, config_class=Blip2Config) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -2053,8 +1799,6 @@ class Blip2VisionModelWithProjection(Blip2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Blip2VisionModelOutput]: r""" - Returns: - Examples: ```python @@ -2123,8 +1867,8 @@ class Blip2VisionModelWithProjection(Blip2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP-2 Model for generating text given an image and an optional text prompt. The model consists of a vision encoder, Querying Transformer (Q-Former) and a language model. @@ -2136,8 +1880,7 @@ class Blip2VisionModelWithProjection(Blip2PreTrainedModel): Note that Flan-T5 checkpoints cannot be cast to float16. They are pre-trained using bfloat16. - """, - BLIP_2_START_DOCSTRING, + """ ) class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin): config_class = Blip2Config @@ -2213,8 +1956,7 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin): if hasattr(self.language_model, "_hf_hook"): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility - @add_start_docstrings_to_model_forward(BLIP_2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Blip2ForConditionalGenerationModelOutput, config_class=Blip2VisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -2231,7 +1973,18 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Blip2ForConditionalGenerationModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. Examples: @@ -2499,13 +2252,12 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin): return outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BLIP-2 Model with a vision and text projector, and a classification head on top. The model is used in the context of image-text retrieval. Given an image and a text, the model returns the probability of the text being relevant to the image. - """, - BLIP_2_START_DOCSTRING, + """ ) class Blip2ForImageTextRetrieval(Blip2PreTrainedModel): main_input_name = "pixel_values" @@ -2539,8 +2291,7 @@ class Blip2ForImageTextRetrieval(Blip2PreTrainedModel): def set_input_embeddings(self, value): self.embeddings.word_embeddings = value - @add_start_docstrings_to_model_forward(BLIP2_IMAGE_TEXT_RETRIEVAL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Blip2ImageTextMatchingModelOutput, config_class=Blip2Config) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -2552,7 +2303,15 @@ class Blip2ForImageTextRetrieval(Blip2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Blip2ImageTextMatchingModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`Blip2Processor`]. See [`Blip2Processor.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + use_image_text_matching_head (`bool`, *optional*): + Whether to return the Image-Text Matching or Contrastive scores. Examples: diff --git a/src/transformers/models/bloom/modeling_bloom.py b/src/transformers/models/bloom/modeling_bloom.py index 1f21db9649..43f9ee6c20 100644 --- a/src/transformers/models/bloom/modeling_bloom.py +++ b/src/transformers/models/bloom/modeling_bloom.py @@ -25,7 +25,6 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss from torch.nn import functional as F from ...cache_utils import Cache, DynamicCache, StaticCache -from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_outputs import ( @@ -37,6 +36,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...utils import ( + auto_docstring, is_torch_flex_attn_available, logging, ) @@ -51,9 +51,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "bigscience/bloom-560m" -_CONFIG_FOR_DOC = "BloomConfig" - def build_alibi_tensor(attention_mask: torch.Tensor, num_heads: int, dtype: torch.dtype) -> torch.Tensor: """ @@ -442,6 +439,7 @@ class BloomBlock(nn.Module): return outputs # hidden_states, past_kv, attentions +@auto_docstring class BloomPreTrainedModel(PreTrainedModel): config_class = BloomConfig base_model_prefix = "transformer" @@ -472,94 +470,7 @@ class BloomPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -BLOOM_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BloomConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BLOOM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` - (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Bloom Model transformer outputting raw hidden-states without any specific head on top.", - BLOOM_START_DOCSTRING, -) +@auto_docstring class BloomModel(BloomPreTrainedModel): def __init__(self, config: BloomConfig): super().__init__(config) @@ -591,12 +502,7 @@ class BloomModel(BloomPreTrainedModel): def set_input_embeddings(self, new_embeddings: torch.Tensor): self.word_embeddings = new_embeddings - @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -611,6 +517,19 @@ class BloomModel(BloomPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **deprecated_arguments, ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ if deprecated_arguments.pop("position_ids", False) is not False: # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` warnings.warn( @@ -865,12 +784,11 @@ class BloomModel(BloomPreTrainedModel): return causal_mask -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - BLOOM_START_DOCSTRING, + """ ) class BloomForCausalLM(BloomPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -951,12 +869,7 @@ class BloomForCausalLM(BloomPreTrainedModel, GenerationMixin): ) return model_inputs - @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -973,6 +886,17 @@ class BloomForCausalLM(BloomPreTrainedModel, GenerationMixin): **deprecated_arguments, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -1056,8 +980,8 @@ class BloomForCausalLM(BloomPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Bloom Model transformer with a sequence classification head on top (linear layer). [`BloomForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1068,8 +992,7 @@ class BloomForCausalLM(BloomPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - BLOOM_START_DOCSTRING, + """ ) class BloomForSequenceClassification(BloomPreTrainedModel): def __init__(self, config: BloomConfig): @@ -1081,12 +1004,7 @@ class BloomForSequenceClassification(BloomPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1102,6 +1020,17 @@ class BloomForSequenceClassification(BloomPreTrainedModel): **deprecated_arguments, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1192,13 +1121,7 @@ class BloomForSequenceClassification(BloomPreTrainedModel): ) -@add_start_docstrings( - """ - Bloom Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - BLOOM_START_DOCSTRING, -) +@auto_docstring class BloomForTokenClassification(BloomPreTrainedModel): def __init__(self, config: BloomConfig): super().__init__(config) @@ -1217,12 +1140,7 @@ class BloomForTokenClassification(BloomPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1238,6 +1156,17 @@ class BloomForTokenClassification(BloomPreTrainedModel): **deprecated_arguments, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1293,13 +1222,7 @@ class BloomForTokenClassification(BloomPreTrainedModel): ) -@add_start_docstrings( - """ - The BLOOM Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - BLOOM_START_DOCSTRING, -) +@auto_docstring class BloomForQuestionAnswering(BloomPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1309,7 +1232,7 @@ class BloomForQuestionAnswering(BloomPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BLOOM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1324,14 +1247,17 @@ class BloomForQuestionAnswering(BloomPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py b/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py index 7af3213854..68d5b2c0f8 100644 --- a/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py +++ b/src/transformers/models/bridgetower/image_processing_bridgetower_fast.py @@ -17,8 +17,6 @@ from typing import Dict, Iterable, Optional, Tuple, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -31,7 +29,7 @@ from ...image_processing_utils_fast import ( reorder_images, ) from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling -from ...utils import add_start_docstrings, is_torch_available, is_torchvision_available, is_torchvision_v2_available +from ...utils import auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available if is_torch_available(): @@ -95,22 +93,21 @@ def get_resize_output_image_size( class BridgeTowerFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): - size_divisor: Optional[int] - do_pad: Optional[bool] - - -@add_start_docstrings( - "Constructs a fast BridgeTower image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, """ + Args: size_divisor (`int`, *optional*, defaults to 32): The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. do_pad (`bool`, *optional*, defaults to `True`): Whether to pad the image to the `(max_height, max_width)` of the images in the batch. Can be overridden by the `do_pad` parameter in the `preprocess` method. - """, -) + """ + + size_divisor: Optional[int] + do_pad: Optional[bool] + + +@auto_docstring class BridgeTowerImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN @@ -129,17 +126,7 @@ class BridgeTowerImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[BridgeTowerFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - size_divisor (`int`, *optional*, defaults to 32): - The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` - is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Whether to pad the image to the `(max_height, max_width)` of the images in the batch. Can be overridden by - the `do_pad` parameter in the `preprocess` method. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[BridgeTowerFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/bridgetower/modeling_bridgetower.py b/src/transformers/models/bridgetower/modeling_bridgetower.py index 524b4caa74..e9ba3f272c 100644 --- a/src/transformers/models/bridgetower/modeling_bridgetower.py +++ b/src/transformers/models/bridgetower/modeling_bridgetower.py @@ -34,96 +34,15 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel, apply_chunking_to_forward from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_bridgetower import BridgeTowerConfig, BridgeTowerTextConfig, BridgeTowerVisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "BridgeTowerConfig" -_CHECKPOINT_FOR_DOC = "BridgeTower/bridgetower-base" _TOKENIZER_FOR_DOC = "RobertaTokenizer" -BRIDGETOWER_START_DOCSTRING = r""" - This model is a PyTorch `torch.nn.Module `_ subclass. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`BridgeTowerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BRIDGETOWER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - [What are token type IDs?](../glossary#token-type-ids) - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`BridgeTowerImageProcessor`]. See - [`BridgeTowerImageProcessor.__call__`] for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - `What are attention masks? <../glossary.html#attention-mask>`__ - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): - Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `pixel_values` into patch embeddings. - - image_token_type_idx (`int`, *optional*): - - The token type ids for images. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - @dataclass class BridgeTowerModelOutput(ModelOutput): """ @@ -1030,12 +949,8 @@ def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_l return incremental_indices.long() + padding_idx +@auto_docstring class BridgeTowerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BridgeTowerConfig base_model_prefix = "bridgetower" supports_gradient_checkpointing = False @@ -1084,9 +999,8 @@ class BridgeTowerVisionModel(BridgeTowerPreTrainedModel): return self.visual(image.type(self.dtype), image_mask, interpolate_pos_encoding) -class BridgeTowerTextModel(BridgeTowerPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz @@ -1097,12 +1011,16 @@ class BridgeTowerTextModel(BridgeTowerPreTrainedModel): `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 - """ - +) +class BridgeTowerTextModel(BridgeTowerPreTrainedModel): config_class = BridgeTowerTextConfig def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1128,6 +1046,7 @@ class BridgeTowerTextModel(BridgeTowerPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) + @auto_docstring # Copied from transformers.models.clap.modeling_clap.ClapTextModel.forward def forward( self, @@ -1145,26 +1064,6 @@ class BridgeTowerTextModel(BridgeTowerPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1260,10 +1159,10 @@ class BridgeTowerTextModel(BridgeTowerPreTrainedModel): ) -@add_start_docstrings( - "The bare BridgeTower Model transformer outputting BridgeTowerModelOutput object without any specific head on" - " top.", - BRIDGETOWER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare BridgeTower Model transformer outputting BridgeTowerModelOutput object without any specific head on + """ ) class BridgeTowerModel(BridgeTowerPreTrainedModel): def __init__(self, config): @@ -1328,8 +1227,7 @@ class BridgeTowerModel(BridgeTowerPreTrainedModel): def set_input_embeddings(self, value): self.text_model.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(BRIDGETOWER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BridgeTowerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1348,6 +1246,11 @@ class BridgeTowerModel(BridgeTowerPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> Union[Tuple[torch.Tensor], BridgeTowerModelOutput]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. + image_token_type_idx (`int`, *optional*): + - The token type ids for images. output_hidden_states (`bool`, *optional*): If set to `True`, hidden states are returned as a list containing the hidden states of text, image, and cross-modal components respectively. i.e. `(hidden_states_text, hidden_states_image, @@ -1357,7 +1260,6 @@ class BridgeTowerModel(BridgeTowerPreTrainedModel): `cross_modal_image_hidden_states` of each brdige layer. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels are currently not supported. - Returns: Examples: @@ -1612,11 +1514,10 @@ class BridgeTowerITMHead(nn.Module): return itm_score -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BridgeTower Model with a language modeling head on top as done during pretraining. - """, - BRIDGETOWER_START_DOCSTRING, + """ ) class BridgeTowerForMaskedLM(BridgeTowerPreTrainedModel): _tied_weights_keys = ["mlm_score.decoder.weight"] @@ -1636,8 +1537,7 @@ class BridgeTowerForMaskedLM(BridgeTowerPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.mlm_score.decoder = new_embeddings - @add_start_docstrings_to_model_forward(BRIDGETOWER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1654,11 +1554,13 @@ class BridgeTowerForMaskedLM(BridgeTowerPreTrainedModel): labels: Optional[torch.LongTensor] = None, ) -> Union[MaskedLMOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - Returns: Examples: @@ -1720,12 +1622,11 @@ class BridgeTowerForMaskedLM(BridgeTowerPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BridgeTower Model transformer with a classifier head on top (a linear layer on top of the final hidden state of the [CLS] token) for image-to-text matching. - """, - BRIDGETOWER_START_DOCSTRING, + """ ) class BridgeTowerForImageAndTextRetrieval(BridgeTowerPreTrainedModel): def __init__(self, config): @@ -1738,8 +1639,7 @@ class BridgeTowerForImageAndTextRetrieval(BridgeTowerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BRIDGETOWER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1756,10 +1656,12 @@ class BridgeTowerForImageAndTextRetrieval(BridgeTowerPreTrainedModel): labels: Optional[torch.LongTensor] = None, ) -> Union[SequenceClassifierOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*): Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match. The pairs with 0 will be skipped for calculation. - Returns: Examples: @@ -1832,11 +1734,10 @@ class BridgeTowerContrastiveHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" BridgeTower Model with a image-text contrastive head on top computing image-text contrastive loss. - """, - BRIDGETOWER_START_DOCSTRING, + """ ) class BridgeTowerForContrastiveLearning(BridgeTowerPreTrainedModel): def __init__(self, config): @@ -1852,8 +1753,7 @@ class BridgeTowerForContrastiveLearning(BridgeTowerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(BRIDGETOWER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BridgeTowerContrastiveOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1870,9 +1770,11 @@ class BridgeTowerForContrastiveLearning(BridgeTowerPreTrainedModel): return_loss: Optional[bool] = None, ) -> Union[BridgeTowerContrastiveOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. - Returns: Examples: diff --git a/src/transformers/models/bros/modeling_bros.py b/src/transformers/models/bros/modeling_bros.py index df39d2a49b..100337d96b 100755 --- a/src/transformers/models/bros/modeling_bros.py +++ b/src/transformers/models/bros/modeling_bros.py @@ -31,100 +31,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_bros import BrosConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "jinho8345/bros-base-uncased" -_CONFIG_FOR_DOC = "BrosConfig" - - -BROS_START_DOCSTRING = r""" - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`BrosConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -BROS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`BrosProcessor`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'): - Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values - (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the - bounding box. - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - bbox_first_token_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - @dataclass class BrosSpadeOutput(ModelOutput): @@ -749,12 +661,8 @@ class BrosRelationExtractor(nn.Module): return relation_score +@auto_docstring class BrosPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = BrosConfig base_model_prefix = "bros" @@ -775,12 +683,13 @@ class BrosPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -@add_start_docstrings( - "The bare Bros Model transformer outputting raw hidden-states without any specific head on top.", - BROS_START_DOCSTRING, -) +@auto_docstring class BrosModel(BrosPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -806,8 +715,7 @@ class BrosModel(BrosPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(BROS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -826,7 +734,10 @@ class BrosModel(BrosPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - Returns: + bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'): + Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values + (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the + bounding box. Examples: @@ -950,13 +861,7 @@ class BrosModel(BrosPreTrainedModel): ) -@add_start_docstrings( - """ - Bros Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - BROS_START_DOCSTRING, -) +@auto_docstring class BrosForTokenClassification(BrosPreTrainedModel): _keys_to_ignore_on_load_unexpected = [r"pooler"] @@ -973,8 +878,7 @@ class BrosForTokenClassification(BrosPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(BROS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -991,8 +895,15 @@ class BrosForTokenClassification(BrosPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'): + Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values + (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the + bounding box. + bbox_first_token_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`: - Returns: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. Examples: @@ -1054,15 +965,14 @@ class BrosForTokenClassification(BrosPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Bros Model with a token classification head on top (initial_token_layers and subsequent_token_layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. The initial_token_classifier is used to predict the first token of each entity, and the subsequent_token_classifier is used to predict the subsequent tokens within an entity. Compared to BrosForTokenClassification, this model is more robust to serialization errors since it predicts next token from one token. - """, - BROS_START_DOCSTRING, + """ ) class BrosSpadeEEForTokenClassification(BrosPreTrainedModel): _keys_to_ignore_on_load_unexpected = [r"pooler"] @@ -1092,8 +1002,7 @@ class BrosSpadeEEForTokenClassification(BrosPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(BROS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BrosSpadeOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1111,7 +1020,19 @@ class BrosSpadeEEForTokenClassification(BrosPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BrosSpadeOutput]: r""" - Returns: + bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'): + Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values + (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the + bounding box. + bbox_first_token_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + initial_token_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for the initial token classification. + subsequent_token_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for the subsequent token classification. Examples: @@ -1200,12 +1121,11 @@ class BrosSpadeEEForTokenClassification(BrosPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Bros Model with a token classification head on top (a entity_linker layer on top of the hidden-states output) e.g. for Entity-Linking. The entity_linker is used to predict intra-entity links (one entity to another entity). - """, - BROS_START_DOCSTRING, + """ ) class BrosSpadeELForTokenClassification(BrosPreTrainedModel): _keys_to_ignore_on_load_unexpected = [r"pooler"] @@ -1224,8 +1144,7 @@ class BrosSpadeELForTokenClassification(BrosPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(BROS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1242,7 +1161,15 @@ class BrosSpadeELForTokenClassification(BrosPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" - Returns: + bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'): + Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values + (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the + bounding box. + bbox_first_token_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. Examples: diff --git a/src/transformers/models/camembert/modeling_camembert.py b/src/transformers/models/camembert/modeling_camembert.py index f1ab156503..733f313078 100644 --- a/src/transformers/models/camembert/modeling_camembert.py +++ b/src/transformers/models/camembert/modeling_camembert.py @@ -26,10 +26,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN, gelu from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - _prepare_4d_attention_mask_for_sdpa, - _prepare_4d_causal_attention_mask_for_sdpa, -) +from ...modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -42,39 +39,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - get_torch_version, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, get_torch_version, logging from .configuration_camembert import CamembertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "almanach/camembert-base" -_CONFIG_FOR_DOC = "CamembertConfig" - - -CAMEMBERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`CamembertConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - # Copied from transformers.models.roberta.modeling_roberta.RobertaEmbeddings with Roberta->Camembert class CamembertEmbeddings(nn.Module): @@ -704,12 +674,8 @@ class CamembertPooler(nn.Module): return pooled_output +@auto_docstring class CamembertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CamembertConfig base_model_prefix = "roberta" supports_gradient_checkpointing = True @@ -735,56 +701,6 @@ class CamembertPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -CAMEMBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.roberta.modeling_roberta.RobertaClassificationHead with Roberta->Camembert class CamembertClassificationHead(nn.Module): """Head for sentence-level classification tasks.""" @@ -840,10 +756,7 @@ class CamembertLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - "The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.", - CAMEMBERT_START_DOCSTRING, -) +@auto_docstring class CamembertModel(CamembertPreTrainedModel): """ @@ -864,6 +777,10 @@ class CamembertModel(CamembertPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaModel.__init__ with Roberta->Camembert def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -892,12 +809,7 @@ class CamembertModel(CamembertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaModel.forward def forward( self, @@ -915,26 +827,6 @@ class CamembertModel(CamembertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, target_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1062,10 +954,7 @@ class CamembertModel(CamembertPreTrainedModel): ) -@add_start_docstrings( - """CamemBERT Model with a `language modeling` head on top.""", - CAMEMBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForMaskedLM with Roberta->Camembert, ROBERTA->CAMEMBERT class CamembertForMaskedLM(CamembertPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1091,15 +980,7 @@ class CamembertForMaskedLM(CamembertPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - expected_output="' Paris'", - expected_loss=0.1, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1116,12 +997,19 @@ class CamembertForMaskedLM(CamembertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1160,12 +1048,11 @@ class CamembertForMaskedLM(CamembertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - CAMEMBERT_START_DOCSTRING, + """ ) # Copied from transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification with Roberta->Camembert, ROBERTA->CAMEMBERT class CamembertForSequenceClassification(CamembertPreTrainedModel): @@ -1180,14 +1067,7 @@ class CamembertForSequenceClassification(CamembertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="cardiffnlp/twitter-roberta-base-emotion", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'optimism'", - expected_loss=0.08, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1202,6 +1082,15 @@ class CamembertForSequenceClassification(CamembertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1260,13 +1149,7 @@ class CamembertForSequenceClassification(CamembertPreTrainedModel): ) -@add_start_docstrings( - """ - CamemBERT Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - CAMEMBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForMultipleChoice with Roberta->Camembert, ROBERTA->CAMEMBERT class CamembertForMultipleChoice(CamembertPreTrainedModel): def __init__(self, config): @@ -1279,14 +1162,7 @@ class CamembertForMultipleChoice(CamembertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1301,10 +1177,35 @@ class CamembertForMultipleChoice(CamembertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1355,13 +1256,7 @@ class CamembertForMultipleChoice(CamembertPreTrainedModel): ) -@add_start_docstrings( - """ - CamemBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - CAMEMBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForTokenClassification with Roberta->Camembert, ROBERTA->CAMEMBERT class CamembertForTokenClassification(CamembertPreTrainedModel): def __init__(self, config): @@ -1378,14 +1273,7 @@ class CamembertForTokenClassification(CamembertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="Jean-Baptiste/roberta-large-ner-english", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="['O', 'ORG', 'ORG', 'O', 'O', 'O', 'O', 'O', 'LOC', 'O', 'LOC', 'LOC']", - expected_loss=0.01, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1400,6 +1288,15 @@ class CamembertForTokenClassification(CamembertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1441,13 +1338,7 @@ class CamembertForTokenClassification(CamembertPreTrainedModel): ) -@add_start_docstrings( - """ - CamemBERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits` - """, - CAMEMBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForQuestionAnswering with Roberta->Camembert, ROBERTA->CAMEMBERT class CamembertForQuestionAnswering(CamembertPreTrainedModel): def __init__(self, config): @@ -1460,14 +1351,7 @@ class CamembertForQuestionAnswering(CamembertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="deepset/roberta-base-squad2", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="' puppet'", - expected_loss=0.86, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1483,14 +1367,15 @@ class CamembertForQuestionAnswering(CamembertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1543,8 +1428,10 @@ class CamembertForQuestionAnswering(CamembertPreTrainedModel): ) -@add_start_docstrings( - """CamemBERT Model with a `language modeling` head on top for CLM fine-tuning.""", CAMEMBERT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + CamemBERT Model with a `language modeling` head on top for CLM fine-tuning. + """ ) # Copied from transformers.models.roberta.modeling_roberta.RobertaForCausalLM with Roberta->Camembert, ROBERTA->CAMEMBERT, FacebookAI/roberta-base->almanach/camembert-base class CamembertForCausalLM(CamembertPreTrainedModel, GenerationMixin): @@ -1568,8 +1455,7 @@ class CamembertForCausalLM(CamembertPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(CAMEMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1589,31 +1475,19 @@ class CamembertForCausalLM(CamembertPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: diff --git a/src/transformers/models/canine/modeling_canine.py b/src/transformers/models/canine/modeling_canine.py index 7a699f6ca9..ae0bed5bd3 100644 --- a/src/transformers/models/canine/modeling_canine.py +++ b/src/transformers/models/canine/modeling_canine.py @@ -36,21 +36,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_canine import CanineConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/canine-s" -_CONFIG_FOR_DOC = "CanineConfig" - # Support up to 16 hash functions. _PRIMES = [31, 43, 59, 61, 73, 97, 103, 113, 137, 149, 157, 173, 181, 193, 211, 223] @@ -880,12 +871,8 @@ class CanineOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class CaninePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CanineConfig load_tf_weights = load_tf_weights_in_canine base_model_prefix = "canine" @@ -908,73 +895,13 @@ class CaninePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -CANINE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`CanineConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CANINE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare CANINE Model transformer outputting raw hidden-states without any specific head on top.", - CANINE_START_DOCSTRING, -) +@auto_docstring class CanineModel(CaninePreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config shallow_config = copy.deepcopy(config) @@ -1081,12 +1008,7 @@ class CanineModel(CaninePreTrainedModel): # `repeated`: [batch_size, char_seq_len, molecule_hidden_size] return torch.cat([repeated, remainder_repeated], dim=-2) - @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CanineModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1246,12 +1168,11 @@ class CanineModel(CaninePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" CANINE Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - CANINE_START_DOCSTRING, + """ ) class CanineForSequenceClassification(CaninePreTrainedModel): def __init__(self, config): @@ -1265,12 +1186,7 @@ class CanineForSequenceClassification(CaninePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1343,13 +1259,7 @@ class CanineForSequenceClassification(CaninePreTrainedModel): ) -@add_start_docstrings( - """ - CANINE Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - CANINE_START_DOCSTRING, -) +@auto_docstring class CanineForMultipleChoice(CaninePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1361,12 +1271,7 @@ class CanineForMultipleChoice(CaninePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1381,6 +1286,30 @@ class CanineForMultipleChoice(CaninePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1434,13 +1363,7 @@ class CanineForMultipleChoice(CaninePreTrainedModel): ) -@add_start_docstrings( - """ - CANINE Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - CANINE_START_DOCSTRING, -) +@auto_docstring class CanineForTokenClassification(CaninePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1453,8 +1376,7 @@ class CanineForTokenClassification(CaninePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1472,8 +1394,6 @@ class CanineForTokenClassification(CaninePreTrainedModel): labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Example: ```python @@ -1540,13 +1460,7 @@ class CanineForTokenClassification(CaninePreTrainedModel): ) -@add_start_docstrings( - """ - CANINE Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - CANINE_START_DOCSTRING, -) +@auto_docstring class CanineForQuestionAnswering(CaninePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1558,14 +1472,7 @@ class CanineForQuestionAnswering(CaninePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CANINE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="Splend1dchan/canine-c-squad", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'nice puppet'", - expected_loss=8.81, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1580,16 +1487,6 @@ class CanineForQuestionAnswering(CaninePreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.canine( diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 3928e88ae0..624450371b 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -27,23 +27,17 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - BaseModelOutputWithPast, - CausalLMOutputWithPast, -) +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import ALL_LAYERNORM_LAYERS from ...utils import ( LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, is_torch_flex_attn_available, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from .configuration_chameleon import ChameleonConfig, ChameleonVQVAEConfig @@ -56,12 +50,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ChameleonConfig" -_CHECKPOINT_FOR_DOC = "meta/chameleon-7b" -_EXPECTED_OUTPUT_SHAPE = [1, 7, 4096] -_SEQ_CLASS_EXPECTED_LOSS = 1.03 -_SEQ_CLASS_EXPECTED_OUTPUT = "'LABEL_0'" - # Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Chameleon class ChameleonRMSNorm(nn.Module): @@ -829,27 +817,7 @@ class ChameleonImageVocabularyMapping: return img_tokens.to(device) -CHAMELEON_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ChameleonConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare chameleon Model outputting raw hidden-states without any specific head on top.", - CHAMELEON_START_DOCSTRING, -) +@auto_docstring class ChameleonPreTrainedModel(PreTrainedModel): config_class = ChameleonConfig base_model_prefix = "model" @@ -882,29 +850,12 @@ class ChameleonPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -CHAMELEON_VQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ChameleonVQVAEConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The VQ-VAE model used in Chameleon for encoding/decoding images into discrete tokens. +@auto_docstring( + custom_intro=""" + The VQ-VAE model used in Chameleon for encoding/decoding images into discrete tokens. This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv Taigman](https://arxiv.org/abs/2203.13131). - """, - CHAMELEON_VQ_START_DOCSTRING, + """ ) class ChameleonVQVAE(ChameleonPreTrainedModel): config_class = ChameleonVQVAEConfig @@ -926,87 +877,8 @@ class ChameleonVQVAE(ChameleonPreTrainedModel): return quant, emb_loss, indices -CHAMELEON_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`ChameleonImageProcessor.__call__`] for details. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Should always be a [`~cache_utils.Cache`] instance and the model will output the same cache instance. - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare chameleon Model outputting raw hidden-states without any specific head on top.", - CHAMELEON_START_DOCSTRING, -) +@auto_docstring class ChameleonModel(ChameleonPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ChameleonDecoderLayer`] - - Args: - config: ChameleonConfig - """ - def __init__(self, config: ChameleonConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1047,13 +919,7 @@ class ChameleonModel(ChameleonPreTrainedModel): bpe_toks = bpe_toks.view(batch_size, -1) return bpe_toks - @add_start_docstrings_to_model_forward(CHAMELEON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1314,9 +1180,10 @@ class ChameleonModel(ChameleonPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - "Chameleon Model with a head on top used for outputting logits for next token prediction.", - CHAMELEON_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Chameleon Model with a head on top used for outputting logits for next token prediction. + """ ) class ChameleonForConditionalGeneration(ChameleonPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1349,8 +1216,7 @@ class ChameleonForConditionalGeneration(ChameleonPreTrainedModel, GenerationMixi return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(CHAMELEON_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1368,12 +1234,10 @@ class ChameleonForConditionalGeneration(ChameleonPreTrainedModel, GenerationMixi **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/chinese_clip/image_processing_chinese_clip_fast.py b/src/transformers/models/chinese_clip/image_processing_chinese_clip_fast.py index a1cb38b8a2..6ced07e0c2 100644 --- a/src/transformers/models/chinese_clip/image_processing_chinese_clip_fast.py +++ b/src/transformers/models/chinese_clip/image_processing_chinese_clip_fast.py @@ -14,15 +14,12 @@ # limitations under the License. """Fast Image processor class for Chinese-CLIP.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast ChineseCLIP image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class ChineseCLIPImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN diff --git a/src/transformers/models/chinese_clip/modeling_chinese_clip.py b/src/transformers/models/chinese_clip/modeling_chinese_clip.py index 647e8f1c24..bc1421e715 100644 --- a/src/transformers/models/chinese_clip/modeling_chinese_clip.py +++ b/src/transformers/models/chinese_clip/modeling_chinese_clip.py @@ -31,23 +31,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_chinese_clip import ChineseCLIPConfig, ChineseCLIPTextConfig, ChineseCLIPVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "OFA-Sys/chinese-clip-vit-base-patch16" -_CONFIG_FOR_DOC = "ChineseCLIPConfig" - # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/CLIP.html # Copied from transformers.models.clip.modeling_clip.contrastive_loss @@ -733,12 +722,8 @@ class ChineseCLIPTextPooler(nn.Module): return pooled_output +@auto_docstring class ChineseCLIPPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ChineseCLIPConfig base_model_prefix = "chinese_clip" supports_gradient_checkpointing = True @@ -791,131 +776,6 @@ class ChineseCLIPPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -CHINESE_CLIP_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ChineseCLIPConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CHINESE_CLIP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CHINESE_CLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`ChineseCLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CHINESE_CLIP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`ChineseCLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->ChineseCLIPText class ChineseCLIPTextEncoder(nn.Module): def __init__(self, config): @@ -1098,8 +958,7 @@ class ChineseCLIPVisionTransformer(nn.Module): self.encoder = ChineseCLIPVisionEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(CHINESE_CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=ChineseCLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1108,9 +967,6 @@ class ChineseCLIPVisionTransformer(nn.Module): interpolate_pos_encoding: bool = False, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1145,9 +1001,10 @@ class ChineseCLIPVisionTransformer(nn.Module): ) -@add_start_docstrings( - "The text model from CHINESE_CLIP without any head or projection on top.", - CHINESE_CLIP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text model from CHINESE_CLIP without any head or projection on top. + """ ) class ChineseCLIPTextModel(ChineseCLIPPreTrainedModel): """ @@ -1166,6 +1023,10 @@ class ChineseCLIPTextModel(ChineseCLIPPreTrainedModel): _no_split_modules = ["ChineseCLIPTextEmbeddings"] def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1191,12 +1052,7 @@ class ChineseCLIPTextModel(ChineseCLIPPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(CHINESE_CLIP_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1213,26 +1069,6 @@ class ChineseCLIPTextModel(ChineseCLIPPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1328,9 +1164,10 @@ class ChineseCLIPTextModel(ChineseCLIPPreTrainedModel): ) -@add_start_docstrings( - """The vision model from CHINESE_CLIP without any head or projection on top.""", - CHINESE_CLIP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from CHINESE_CLIP without any head or projection on top. + """ ) class ChineseCLIPVisionModel(ChineseCLIPPreTrainedModel): config_class = ChineseCLIPVisionConfig @@ -1346,8 +1183,7 @@ class ChineseCLIPVisionModel(ChineseCLIPPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(CHINESE_CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=ChineseCLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1357,8 +1193,6 @@ class ChineseCLIPVisionModel(ChineseCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -1389,7 +1223,7 @@ class ChineseCLIPVisionModel(ChineseCLIPPreTrainedModel): ) -@add_start_docstrings(CHINESE_CLIP_START_DOCSTRING) +@auto_docstring class ChineseCLIPModel(ChineseCLIPPreTrainedModel): config_class = ChineseCLIPConfig @@ -1425,7 +1259,7 @@ class ChineseCLIPModel(ChineseCLIPPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CHINESE_CLIP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1475,7 +1309,7 @@ class ChineseCLIPModel(ChineseCLIPPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(CHINESE_CLIP_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1527,8 +1361,7 @@ class ChineseCLIPModel(ChineseCLIPPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(CHINESE_CLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ChineseCLIPOutput, config_class=ChineseCLIPConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1543,7 +1376,8 @@ class ChineseCLIPModel(ChineseCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ChineseCLIPOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: diff --git a/src/transformers/models/clap/modeling_clap.py b/src/transformers/models/clap/modeling_clap.py index 1988bb95c6..a088490717 100644 --- a/src/transformers/models/clap/modeling_clap.py +++ b/src/transformers/models/clap/modeling_clap.py @@ -31,21 +31,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, meshgrid, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_clap import ClapAudioConfig, ClapConfig, ClapTextConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "laion/clap-htsat-fused" - # Adapted from: https://github.com/LAION-AI/CLAP/blob/6ad05a971ba0622f6acee8c41993e0d02bbed639/src/open_clip/utils.py#L191 def interpolate(hidden_states, ratio): @@ -1017,109 +1008,6 @@ class ClapAudioEncoder(nn.Module): ) -CLAP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ClapConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CLAP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CLAP_AUDIO_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Input audio features. This should be returned by the [`ClapFeatureExtractor`] class that you can also - retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. - is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): - Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance - the features. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CLAP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Input audio features. This should be returned by the [`ClapFeatureExtractor`] class that you can also - retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class ClapProjectionLayer(nn.Module): def __init__(self, config: Union[ClapAudioConfig, ClapTextConfig]): super().__init__() @@ -1663,12 +1551,8 @@ class ClapTextPooler(nn.Module): return pooled_output +@auto_docstring class ClapPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ClapConfig base_model_prefix = "clap" supports_gradient_checkpointing = False @@ -1709,8 +1593,7 @@ class ClapAudioModel(ClapPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.audio_encoder.patch_embed.proj - @add_start_docstrings_to_model_forward(CLAP_AUDIO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=ClapAudioConfig) + @auto_docstring def forward( self, input_features: Optional[torch.FloatTensor] = None, @@ -1720,7 +1603,12 @@ class ClapAudioModel(ClapPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Input audio features. This should be returned by the [`ClapFeatureExtractor`] class that you can also + retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. + is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): + Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance + the features. Examples: @@ -1754,9 +1642,8 @@ class ClapAudioModel(ClapPreTrainedModel): ) -class ClapTextModel(ClapPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz @@ -1767,12 +1654,16 @@ class ClapTextModel(ClapPreTrainedModel): `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 - """ - +) +class ClapTextModel(ClapPreTrainedModel): config_class = ClapTextConfig def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1790,6 +1681,7 @@ class ClapTextModel(ClapPreTrainedModel): def set_input_embeddings(self, value): self.embeddings.word_embeddings = value + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1806,26 +1698,6 @@ class ClapTextModel(ClapPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1921,7 +1793,7 @@ class ClapTextModel(ClapPreTrainedModel): ) -@add_start_docstrings(CLAP_START_DOCSTRING) +@auto_docstring class ClapModel(ClapPreTrainedModel): config_class = ClapConfig @@ -1957,7 +1829,7 @@ class ClapModel(ClapPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CLAP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -2005,7 +1877,7 @@ class ClapModel(ClapPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(CLAP_AUDIO_INPUTS_DOCSTRING) + @auto_docstring def get_audio_features( self, input_features: Optional[torch.Tensor] = None, @@ -2016,6 +1888,13 @@ class ClapModel(ClapPreTrainedModel): return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Input audio features. This should be returned by the [`ClapFeatureExtractor`] class that you can also + retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. + is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): + Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance + the features. + Returns: audio_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The audio embeddings obtained by applying the projection layer to the pooled output of [`ClapAudioModel`]. @@ -2051,8 +1930,7 @@ class ClapModel(ClapPreTrainedModel): return audio_features - @add_start_docstrings_to_model_forward(CLAP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ClapOutput, config_class=ClapConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2066,7 +1944,14 @@ class ClapModel(ClapPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ClapOutput]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Input audio features. This should be returned by the [`ClapFeatureExtractor`] class that you can also + retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. + is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): + Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance + the features. Examples: @@ -2149,12 +2034,7 @@ class ClapModel(ClapPreTrainedModel): ) -@add_start_docstrings( - """ - CLAP Text Model with a projection layer on top (a linear layer on top of the pooled output). - """, - CLAP_START_DOCSTRING, -) +@auto_docstring class ClapTextModelWithProjection(ClapPreTrainedModel): config_class = ClapTextConfig @@ -2171,8 +2051,7 @@ class ClapTextModelWithProjection(ClapPreTrainedModel): def set_input_embeddings(self, value): self.text_model.embeddings.word_embeddings = value - @add_start_docstrings_to_model_forward(CLAP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ClapTextModelOutput, config_class=ClapTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2183,8 +2062,6 @@ class ClapTextModelWithProjection(ClapPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ClapTextModelOutput]: r""" - Returns: - Examples: ```python @@ -2225,12 +2102,7 @@ class ClapTextModelWithProjection(ClapPreTrainedModel): ) -@add_start_docstrings( - """ - CLAP Audio Model with a projection layer on top (a linear layer on top of the pooled output). - """, - CLAP_START_DOCSTRING, -) +@auto_docstring class ClapAudioModelWithProjection(ClapPreTrainedModel): config_class = ClapAudioConfig main_input_name = "input_features" @@ -2245,8 +2117,7 @@ class ClapAudioModelWithProjection(ClapPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.audio_model.audio_encoder.patch_embed.proj - @add_start_docstrings_to_model_forward(CLAP_AUDIO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ClapAudioModelOutput, config_class=ClapAudioConfig) + @auto_docstring def forward( self, input_features: Optional[torch.FloatTensor] = None, @@ -2256,7 +2127,12 @@ class ClapAudioModelWithProjection(ClapPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ClapAudioModelOutput]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Input audio features. This should be returned by the [`ClapFeatureExtractor`] class that you can also + retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. + is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): + Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance + the features. Examples: diff --git a/src/transformers/models/clip/image_processing_clip_fast.py b/src/transformers/models/clip/image_processing_clip_fast.py index ec83805125..accd41475d 100644 --- a/src/transformers/models/clip/image_processing_clip_fast.py +++ b/src/transformers/models/clip/image_processing_clip_fast.py @@ -14,15 +14,12 @@ # limitations under the License. """Fast Image processor class for CLIP.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast CLIP image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class CLIPImageProcessorFast(BaseImageProcessorFast): # To be checked against the slow image processor # None values left after checking can be removed diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 4136aabf1d..1ddf6bf8f7 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -25,29 +25,12 @@ from ...activations import ACT2FN from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging, torch_int from .configuration_clip import CLIPConfig, CLIPTextConfig, CLIPVisionConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "CLIPConfig" -_CHECKPOINT_FOR_DOC = "openai/clip-vit-base-patch32" - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "openai/clip-vit-base-patch32" -_IMAGE_CLASS_EXPECTED_OUTPUT = "LABEL_0" - # contrastive loss function, adapted from # https://sachinruk.github.io/blog/2021-03-07-clip.html @@ -460,12 +443,8 @@ class CLIPEncoderLayer(nn.Module): return outputs +@auto_docstring class CLIPPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CLIPConfig base_model_prefix = "clip" supports_gradient_checkpointing = True @@ -529,110 +508,6 @@ class CLIPPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -CLIP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`CLIPConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CLIP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CLIP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class CLIPEncoder(nn.Module): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a @@ -745,8 +620,7 @@ class CLIPTextTransformer(nn.Module): self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -755,10 +629,6 @@ class CLIPTextTransformer(nn.Module): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -824,9 +694,10 @@ class CLIPTextTransformer(nn.Module): ) -@add_start_docstrings( - """The text model from CLIP without any head or projection on top.""", - CLIP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text model from CLIP without any head or projection on top. + """ ) class CLIPTextModel(CLIPPreTrainedModel): config_class = CLIPTextConfig @@ -846,8 +717,7 @@ class CLIPTextModel(CLIPPreTrainedModel): self.text_model.embeddings.token_embedding = value @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -857,8 +727,6 @@ class CLIPTextModel(CLIPPreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: r""" - Returns: - Examples: ```python @@ -895,8 +763,7 @@ class CLIPVisionTransformer(nn.Module): self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -904,10 +771,6 @@ class CLIPVisionTransformer(nn.Module): output_hidden_states: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = False, ) -> BaseModelOutputWithPooling: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -937,9 +800,10 @@ class CLIPVisionTransformer(nn.Module): ) -@add_start_docstrings( - """The vision model from CLIP without any head or projection on top.""", - CLIP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from CLIP without any head or projection on top. + """ ) class CLIPVisionModel(CLIPPreTrainedModel): config_class = CLIPVisionConfig @@ -956,8 +820,7 @@ class CLIPVisionModel(CLIPPreTrainedModel): return self.vision_model.embeddings.patch_embedding @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -966,9 +829,7 @@ class CLIPVisionModel(CLIPPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> BaseModelOutputWithPooling: r""" - Returns: - - Examples: + Example: ```python >>> from PIL import Image @@ -996,7 +857,7 @@ class CLIPVisionModel(CLIPPreTrainedModel): ) -@add_start_docstrings(CLIP_START_DOCSTRING) +@auto_docstring class CLIPModel(CLIPPreTrainedModel): config_class = CLIPConfig _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer", "CLIPVisionEmbeddings"] @@ -1036,7 +897,7 @@ class CLIPModel(CLIPPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1080,7 +941,7 @@ class CLIPModel(CLIPPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1129,8 +990,7 @@ class CLIPModel(CLIPPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CLIPOutput, config_class=CLIPConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1143,7 +1003,8 @@ class CLIPModel(CLIPPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> CLIPOutput: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: @@ -1218,12 +1079,7 @@ class CLIPModel(CLIPPreTrainedModel): ) -@add_start_docstrings( - """ - CLIP Text Model with a projection layer on top (a linear layer on top of the pooled output). - """, - CLIP_START_DOCSTRING, -) +@auto_docstring class CLIPTextModelWithProjection(CLIPPreTrainedModel): config_class = CLIPTextConfig @@ -1247,8 +1103,7 @@ class CLIPTextModelWithProjection(CLIPPreTrainedModel): self.text_model.embeddings.token_embedding = value @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CLIPTextModelOutput, config_class=CLIPTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1258,8 +1113,6 @@ class CLIPTextModelWithProjection(CLIPPreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> CLIPTextModelOutput: r""" - Returns: - Examples: ```python @@ -1292,12 +1145,7 @@ class CLIPTextModelWithProjection(CLIPPreTrainedModel): ) -@add_start_docstrings( - """ - CLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). - """, - CLIP_START_DOCSTRING, -) +@auto_docstring class CLIPVisionModelWithProjection(CLIPPreTrainedModel): config_class = CLIPVisionConfig main_input_name = "pixel_values" @@ -1317,8 +1165,7 @@ class CLIPVisionModelWithProjection(CLIPPreTrainedModel): return self.vision_model.embeddings.patch_embedding @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CLIPVisionModelOutput, config_class=CLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1327,8 +1174,6 @@ class CLIPVisionModelWithProjection(CLIPPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> CLIPVisionModelOutput: r""" - Returns: - Examples: ```python @@ -1365,12 +1210,11 @@ class CLIPVisionModelWithProjection(CLIPPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" CLIP vision encoder with an image classification head on top (a linear layer on top of the pooled final hidden states of the patch tokens) e.g. for ImageNet. - """, - CLIP_START_DOCSTRING, + """ ) class CLIPForImageClassification(CLIPPreTrainedModel): main_input_name = "pixel_values" @@ -1391,13 +1235,7 @@ class CLIPForImageClassification(CLIPPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/clipseg/modeling_clipseg.py b/src/transformers/models/clipseg/modeling_clipseg.py index d626914d72..2321932812 100644 --- a/src/transformers/models/clipseg/modeling_clipseg.py +++ b/src/transformers/models/clipseg/modeling_clipseg.py @@ -27,23 +27,13 @@ from ...activations import ACT2FN from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_clipseg import CLIPSegConfig, CLIPSegTextConfig, CLIPSegVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "CIDAS/clipseg-rd64-refined" - - # contrastive loss function, adapted from # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/CLIP.html def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: @@ -434,12 +424,8 @@ class CLIPSegEncoderLayer(nn.Module): return outputs +@auto_docstring class CLIPSegPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CLIPSegConfig base_model_prefix = "clip" supports_gradient_checkpointing = True @@ -486,106 +472,6 @@ class CLIPSegPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -CLIPSEG_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`CLIPSegConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CLIPSEG_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CLIPSEG_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `True`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -CLIPSEG_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `True`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->CLIPSeg class CLIPSegEncoder(nn.Module): """ @@ -696,9 +582,7 @@ class CLIPSegTextTransformer(nn.Module): # For `pooled_output` computation self.eos_token_id = config.eos_token_id - @add_start_docstrings_to_model_forward(CLIPSEG_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPSegTextConfig) - # Adapted from transformers.models.clip.modeling_clip.CLIPTextTransformer.forward with clip->clipseg, CLIP->CLIPSeg + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -708,10 +592,6 @@ class CLIPSegTextTransformer(nn.Module): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -798,8 +678,7 @@ class CLIPSegTextModel(CLIPSegPreTrainedModel): def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value - @add_start_docstrings_to_model_forward(CLIPSEG_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPSegTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -810,8 +689,6 @@ class CLIPSegTextModel(CLIPSegPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -848,8 +725,7 @@ class CLIPSegVisionTransformer(nn.Module): self.encoder = CLIPSegEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(CLIPSEG_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPSegVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor], @@ -858,10 +734,6 @@ class CLIPSegVisionTransformer(nn.Module): return_dict: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = True, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -906,8 +778,7 @@ class CLIPSegVisionModel(CLIPSegPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(CLIPSEG_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPSegVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -917,8 +788,6 @@ class CLIPSegVisionModel(CLIPSegPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -947,7 +816,7 @@ class CLIPSegVisionModel(CLIPSegPreTrainedModel): ) -@add_start_docstrings(CLIPSEG_START_DOCSTRING) +@auto_docstring class CLIPSegModel(CLIPSegPreTrainedModel): config_class = CLIPSegConfig @@ -983,7 +852,7 @@ class CLIPSegModel(CLIPSegPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CLIPSEG_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1030,7 +899,7 @@ class CLIPSegModel(CLIPSegPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(CLIPSEG_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1081,8 +950,7 @@ class CLIPSegModel(CLIPSegPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(CLIPSEG_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CLIPSegOutput, config_class=CLIPSegConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1096,7 +964,8 @@ class CLIPSegModel(CLIPSegPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, CLIPSegOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: @@ -1334,11 +1203,10 @@ class CLIPSegDecoder(CLIPSegPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation. - """, - CLIPSEG_START_DOCSTRING, + """ ) class CLIPSegForImageSegmentation(CLIPSegPreTrainedModel): config_class = CLIPSegConfig @@ -1385,8 +1253,7 @@ class CLIPSegForImageSegmentation(CLIPSegPreTrainedModel): return conditional_embeddings - @add_start_docstrings_to_model_forward(CLIPSEG_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CLIPSegImageSegmentationOutput, config_class=CLIPSegTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.FloatTensor] = None, @@ -1406,8 +1273,11 @@ class CLIPSegForImageSegmentation(CLIPSegPreTrainedModel): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - - Returns: + conditional_pixel_values (`torch.FloatTensor`, *optional*): + The pixel values of the conditional images. + conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*): + The conditional embeddings for the query images. If provided, the model will use this instead of computing + the embeddings from the conditional_pixel_values. Examples: diff --git a/src/transformers/models/clvp/modeling_clvp.py b/src/transformers/models/clvp/modeling_clvp.py index 136e8d18b2..677858fe80 100644 --- a/src/transformers/models/clvp/modeling_clvp.py +++ b/src/transformers/models/clvp/modeling_clvp.py @@ -38,10 +38,8 @@ from ...modeling_utils import PreTrainedModel from ...pytorch_utils import Conv1D, isin_mps_friendly from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_clvp import ( ClvpConfig, @@ -52,8 +50,6 @@ from .configuration_clvp import ( logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "susnato/clvp_dev" - # Copied from transformers.models.clip.modeling_clip.contrastive_loss def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: @@ -807,12 +803,8 @@ class ClvpConditioningEncoder(nn.Module): return torch.concat([mel_spec, text_embeds], dim=1) +@auto_docstring class ClvpPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ClvpConfig base_model_prefix = "clvp" supports_gradient_checkpointing = True @@ -851,122 +843,6 @@ class ClvpPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -CLVP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ClvpConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -CLVP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, time_dim)`): - Indicates log mel-spectrogram representations for audio returned by [`ClvpFeatureExtractor`]. - conditioning_encoder_inputs_embeds (`torch.FloatTensor`, *optional*): - inputs_embeds for `ClvpConditioningEncoder`. Can be used in place of `input_ids`. - text_encoder_inputs_embeds (`torch.FloatTensor`, *optional*): - inputs_embeds for the text encoder model passed in place of `input_ids`. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding text token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -CLVP_DECODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`): - Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see - `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have - their past given to this model should not be passed as `input_ids` as they have already been computed. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If `past_key_values` is used, `attention_mask` needs to contain the masking strategy that was used for - `past_key_values`. In other words, the `attention_mask` always has to have the length: - `len(past_key_values) + len(input_ids)` - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, input_ids_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class ClvpEncoder(ClvpPreTrainedModel): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a @@ -1158,7 +1034,7 @@ class ClvpDecoder(ClvpPreTrainedModel): for layer, heads in heads_to_prune.items(): self.layers[layer].attn.prune_heads(heads) - @add_start_docstrings_to_model_forward(CLVP_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1302,10 +1178,7 @@ class ClvpDecoder(ClvpPreTrainedModel): ) -@add_start_docstrings( - "The bare Clvp decoder model outputting raw hidden-states without any specific head on top.", - CLVP_START_DOCSTRING, -) +@auto_docstring class ClvpModel(ClvpPreTrainedModel): def __init__(self, config: ClvpDecoderConfig): super().__init__(config) @@ -1324,7 +1197,7 @@ class ClvpModel(ClvpPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(CLVP_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1373,9 +1246,10 @@ class ClvpModel(ClvpPreTrainedModel): ) -@add_start_docstrings( - "The CLVP decoder model with a language modelling head on top.", - CLVP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The CLVP decoder model with a language modelling head on top. + """ ) class ClvpForCausalLM(ClvpPreTrainedModel, GenerationMixin): def __init__(self, config): @@ -1516,7 +1390,7 @@ class ClvpForCausalLM(ClvpPreTrainedModel, GenerationMixin): ) return model_inputs - @add_start_docstrings_to_model_forward(CLVP_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1603,11 +1477,10 @@ class ClvpForCausalLM(ClvpPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - "The composite CLVP model with a text encoder, speech encoder and speech decoder model." - "The speech decoder model generates the speech_ids from the text and the text encoder and speech encoder works" - "together to filter out the best speech_ids.", - CLVP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The composite CLVP model with a text encoder, speech encoder and speech decoder model. + """ ) class ClvpModelForConditionalGeneration(ClvpPreTrainedModel, GenerationMixin): config_class = ClvpConfig @@ -1831,8 +1704,7 @@ class ClvpModelForConditionalGeneration(ClvpPreTrainedModel, GenerationMixin): return outputs[0] - @add_start_docstrings_to_model_forward(CLVP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ClvpOutput, config_class=ClvpConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1846,7 +1718,14 @@ class ClvpModelForConditionalGeneration(ClvpPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, ClvpOutput]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, time_dim)`): + Indicates log mel-spectrogram representations for audio returned by [`ClvpFeatureExtractor`]. + conditioning_encoder_inputs_embeds (`torch.FloatTensor`, *optional*): + inputs_embeds for `ClvpConditioningEncoder`. Can be used in place of `input_ids`. + text_encoder_inputs_embeds (`torch.FloatTensor`, *optional*): + inputs_embeds for the text encoder model passed in place of `input_ids`. + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: diff --git a/src/transformers/models/codegen/modeling_codegen.py b/src/transformers/models/codegen/modeling_codegen.py index 5dcbf09abe..d421432d77 100644 --- a/src/transformers/models/codegen/modeling_codegen.py +++ b/src/transformers/models/codegen/modeling_codegen.py @@ -27,9 +27,7 @@ from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, logging, ) @@ -44,9 +42,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Salesforce/codegen-2B-mono" -_CONFIG_FOR_DOC = "CodeGenConfig" - # Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: @@ -296,12 +291,8 @@ class CodeGenBlock(nn.Module): return outputs # hidden_states, present, (attentions) +@auto_docstring class CodeGenPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CodeGenConfig base_model_prefix = "transformer" supports_gradient_checkpointing = True @@ -331,93 +322,7 @@ class CodeGenPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -CODEGEN_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`CodeGenConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CODEGEN_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare CodeGen Model transformer outputting raw hidden-states without any specific head on top.", - CODEGEN_START_DOCSTRING, -) +@auto_docstring class CodeGenModel(CodeGenPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -441,12 +346,7 @@ class CodeGenModel(CodeGenPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.wte = new_embeddings - @add_start_docstrings_to_model_forward(CODEGEN_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -463,6 +363,12 @@ class CodeGenModel(CodeGenPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs, # NOOP kwargs, for now ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -711,11 +617,10 @@ class CodeGenModel(CodeGenPreTrainedModel): return causal_mask -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The CodeGen Model transformer with a language modeling head on top. - """, - CODEGEN_START_DOCSTRING, + """ ) class CodeGenForCausalLM(CodeGenPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -734,12 +639,7 @@ class CodeGenForCausalLM(CodeGenPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(CODEGEN_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -758,6 +658,10 @@ class CodeGenForCausalLM(CodeGenPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` diff --git a/src/transformers/models/cohere/modeling_cohere.py b/src/transformers/models/cohere/modeling_cohere.py index b5a3fc6100..9611d92ca6 100644 --- a/src/transformers/models/cohere/modeling_cohere.py +++ b/src/transformers/models/cohere/modeling_cohere.py @@ -42,15 +42,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_cohere import CohereConfig @@ -62,8 +54,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "CohereConfig" - class CohereLayerNorm(nn.Module): def __init__(self, hidden_size=None, eps=1e-5, bias=False): @@ -370,27 +360,7 @@ class CohereDecoderLayer(GradientCheckpointingLayer): return outputs -COHERE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`CohereConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Cohere Model outputting raw hidden-states without any specific head on top.", - COHERE_START_DOCSTRING, -) +@auto_docstring class CoherePreTrainedModel(PreTrainedModel): config_class = CohereConfig base_model_prefix = "model" @@ -419,88 +389,8 @@ class CoherePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -COHERE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Cohere Model outputting raw hidden-states without any specific head on top.", - COHERE_START_DOCSTRING, -) +@auto_docstring class CohereModel(CoherePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`CohereDecoderLayer`] - - Args: - config: CohereConfig - """ - def __init__(self, config: CohereConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -524,7 +414,7 @@ class CohereModel(CoherePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(COHERE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -746,6 +636,7 @@ class CohereModel(CoherePreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class CohereForCausalLM(CoherePreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -781,8 +672,7 @@ class CohereForCausalLM(CoherePreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(COHERE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -799,19 +689,10 @@ class CohereForCausalLM(CoherePreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/cohere/modular_cohere.py b/src/transformers/models/cohere/modular_cohere.py index c4acb43494..a44aebcead 100644 --- a/src/transformers/models/cohere/modular_cohere.py +++ b/src/transformers/models/cohere/modular_cohere.py @@ -51,8 +51,6 @@ from .configuration_cohere import CohereConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "CohereConfig" - class CohereLayerNorm(nn.Module): def __init__(self, hidden_size=None, eps=1e-5, bias=False): @@ -330,19 +328,10 @@ class CohereForCausalLM(LlamaForCausalLM): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/cohere2/modeling_cohere2.py b/src/transformers/models/cohere2/modeling_cohere2.py index da68159e77..689b535f57 100644 --- a/src/transformers/models/cohere2/modeling_cohere2.py +++ b/src/transformers/models/cohere2/modeling_cohere2.py @@ -33,15 +33,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from ...utils.deprecation import deprecate_kwarg from .configuration_cohere2 import Cohere2Config @@ -54,8 +46,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Cohere2Config" - class Cohere2RotaryEmbedding(nn.Module): def __init__(self, config: Cohere2Config, device=None): @@ -390,27 +380,7 @@ class Cohere2DecoderLayer(GradientCheckpointingLayer): return outputs -COHERE2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Cohere2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Cohere2 Model outputting raw hidden-states without any specific head on top.", - COHERE2_START_DOCSTRING, -) +@auto_docstring class Cohere2PreTrainedModel(PreTrainedModel): config_class = Cohere2Config base_model_prefix = "model" @@ -439,87 +409,8 @@ class Cohere2PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -COHERE2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Cohere2 Model outputting raw hidden-states without any specific head on top.", - COHERE2_START_DOCSTRING, -) +@auto_docstring class Cohere2Model(Cohere2PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Cohere2DecoderLayer`] - Args: - config: Cohere2Config - """ - def __init__(self, config: Cohere2Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -543,8 +434,7 @@ class Cohere2Model(Cohere2PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(COHERE2_INPUTS_DOCSTRING) - @deprecate_kwarg("last_cache_position", version="4.53.0") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -739,6 +629,7 @@ class Cohere2Model(Cohere2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Cohere2ForCausalLM(Cohere2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -774,8 +665,7 @@ class Cohere2ForCausalLM(Cohere2PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(COHERE2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -792,19 +682,10 @@ class Cohere2ForCausalLM(Cohere2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/cohere2/modular_cohere2.py b/src/transformers/models/cohere2/modular_cohere2.py index 9a47f49393..256a597a41 100644 --- a/src/transformers/models/cohere2/modular_cohere2.py +++ b/src/transformers/models/cohere2/modular_cohere2.py @@ -26,7 +26,7 @@ from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import rope_config_validation from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import add_start_docstrings_to_model_forward, can_return_tuple, logging +from ...utils import logging from ...utils.deprecation import deprecate_kwarg from ..cohere.modeling_cohere import ( CohereAttention, @@ -41,9 +41,6 @@ from ..cohere.modeling_cohere import ( from ..gemma2.modeling_gemma2 import Gemma2Model -COHERE2_INPUTS_DOCSTRING = None # Will be picked up by modular - - logger = logging.get_logger(__name__) @@ -444,20 +441,11 @@ class Cohere2PreTrainedModel(CoherePreTrainedModel): class Cohere2Model(Gemma2Model): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Cohere2DecoderLayer`] - Args: - config: Cohere2Config - """ - def __init__(self, config: Cohere2Config): super().__init__(config) self.norm = Cohere2LayerNorm(hidden_size=(config.hidden_size), eps=config.layer_norm_eps) self.rotary_emb = Cohere2RotaryEmbedding(config=config) - @can_return_tuple - @add_start_docstrings_to_model_forward(COHERE2_INPUTS_DOCSTRING) - @deprecate_kwarg("last_cache_position", version="4.53.0") def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/colpali/modeling_colpali.py b/src/transformers/models/colpali/modeling_colpali.py index c3a669d4c9..3714a9a6f2 100644 --- a/src/transformers/models/colpali/modeling_colpali.py +++ b/src/transformers/models/colpali/modeling_colpali.py @@ -24,38 +24,16 @@ from transformers import AutoModelForImageTextToText from ...cache_utils import Cache from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring from .configuration_colpali import ColPaliConfig -_CONFIG_FOR_DOC = "ColPaliConfig" - -COLPALI_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ColPaliConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare ColPali model outputting raw hidden-states without any specific head on top.", - COLPALI_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare ColPali model outputting raw hidden-states without any specific head on top. + """ ) +@auto_docstring class ColPaliPreTrainedModel(PreTrainedModel): config_class = ColPaliConfig base_model_prefix = "model" @@ -118,47 +96,8 @@ class ColPaliForRetrievalOutput(ModelOutput): image_hidden_states: Optional[torch.FloatTensor] = None -COLPALI_FOR_RETRIEVAL_INPUT_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`PaliGemmaProcessor`] uses - [`SiglipImageProcessor`] for processing images). If none, ColPali will only process text (query embeddings). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - kwargs (`Dict[str, Any]`, *optional*): - Additional key word arguments passed along to the vlm backbone model. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" In our proposed ColPali approach, we leverage VLMs to construct efficient multi-vector embeddings directly from document images (“screenshots”) for document retrieval. We train the model to maximize the similarity between these document embeddings and the corresponding query embeddings, using the late interaction method @@ -187,8 +126,7 @@ class ColPaliForRetrieval(ColPaliPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(COLPALI_FOR_RETRIEVAL_INPUT_DOCSTRING) - @replace_return_docstrings(output_type=ColPaliForRetrievalOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -199,9 +137,6 @@ class ColPaliForRetrieval(ColPaliPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple, ColPaliForRetrievalOutput]: - r""" - Returns: - """ if "pixel_values" in kwargs: kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype=self.dtype) output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr_fast.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr_fast.py index 0566eb3947..c1448fe612 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr_fast.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr_fast.py @@ -9,8 +9,6 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Union from ...image_processing_utils import BatchFeature, get_size_dict from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, @@ -33,7 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -68,6 +66,26 @@ logger = logging.get_logger(__name__) class ConditionalDetrFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): + Data format of the annotations. One of "coco_detection" or "coco_panoptic". + do_convert_annotations (`bool`, *optional*, defaults to `True`): + Controls whether to convert the annotations to the format expected by the CONDITIONAL_DETR model. Converts the + bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. + Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with zeros. + If `pad_size` is provided, the image will be padded to the specified dimensions. + Otherwise, the image will be padded to the maximum height and width of the batch. + pad_size (`Dict[str, int]`, *optional*): + The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size + provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest + height and width in the batch. + return_segmentation_masks (`bool`, *optional*, defaults to `False`): + Whether to return segmentation masks. + """ + format: Optional[Union[str, AnnotationFormat]] do_convert_annotations: Optional[bool] do_pad: Optional[bool] @@ -275,29 +293,7 @@ def prepare_coco_panoptic_annotation( return new_target -@add_start_docstrings( - "Constructs a fast ConditionalDetr image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the CONDITIONAL_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - """, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class ConditionalDetrImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR @@ -571,9 +567,15 @@ class ConditionalDetrImageProcessorFast(BaseImageProcessorFast): return image, pixel_mask, annotation - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ + @auto_docstring + def preprocess( + self, + images: ImageInput, + annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, + masks_path: Optional[Union[str, pathlib.Path]] = None, + **kwargs: Unpack[ConditionalDetrFastImageProcessorKwargs], + ) -> BatchFeature: + r""" annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: @@ -587,32 +589,9 @@ class ConditionalDetrImageProcessorFast(BaseImageProcessorFast): - "file_name" (`str`): The file name of the image. format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the CONDITIONAL_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. masks_path (`str` or `pathlib.Path`, *optional*): Path to the directory containing the segmentation masks. - """, - ) - def preprocess( - self, - images: ImageInput, - annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, - masks_path: Optional[Union[str, pathlib.Path]] = None, - **kwargs: Unpack[ConditionalDetrFastImageProcessorKwargs], - ) -> BatchFeature: + """ if "pad_and_return_pixel_mask" in kwargs: kwargs["do_pad"] = kwargs.pop("pad_and_return_pixel_mask") logger.warning_once( diff --git a/src/transformers/models/conditional_detr/modeling_conditional_detr.py b/src/transformers/models/conditional_detr/modeling_conditional_detr.py index 13fde28d02..0b63dd3330 100644 --- a/src/transformers/models/conditional_detr/modeling_conditional_detr.py +++ b/src/transformers/models/conditional_detr/modeling_conditional_detr.py @@ -25,15 +25,7 @@ from ...activations import ACT2FN from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttentions, Seq2SeqModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_timm_available, - logging, - replace_return_docstrings, - requires_backends, -) +from ...utils import ModelOutput, auto_docstring, is_timm_available, logging, requires_backends from ...utils.backbone_utils import load_backbone from .configuration_conditional_detr import ConditionalDetrConfig @@ -44,9 +36,6 @@ if is_timm_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ConditionalDetrConfig" -_CHECKPOINT_FOR_DOC = "microsoft/conditional-detr-resnet-50" - @dataclass class ConditionalDetrDecoderOutput(BaseModelOutputWithCrossAttentions): @@ -1031,6 +1020,8 @@ class MLP(nn.Module): return x +@auto_docstring + # Copied from transformers.models.detr.modeling_detr.DetrPreTrainedModel with Detr->ConditionalDetr class ConditionalDetrPreTrainedModel(PreTrainedModel): config_class = ConditionalDetrConfig @@ -1062,61 +1053,6 @@ class ConditionalDetrPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -CONDITIONAL_DETR_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ConditionalDetrConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CONDITIONAL_DETR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`AutoImageProcessor`]. See [`ConditionalDetrImageProcessor.__call__`] - for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): - Not used by default. Can be used to mask object queries. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.detr.modeling_detr.DetrEncoder with Detr->ConditionalDetr,DETR->ConditionalDETR class ConditionalDetrEncoder(ConditionalDetrPreTrainedModel): """ @@ -1433,12 +1369,11 @@ class ConditionalDetrDecoder(ConditionalDetrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The bare Conditional DETR Model (consisting of a backbone and encoder-decoder Transformer) outputting raw hidden-states without any specific head on top. - """, - CONDITIONAL_DETR_START_DOCSTRING, + """ ) class ConditionalDetrModel(ConditionalDetrPreTrainedModel): def __init__(self, config: ConditionalDetrConfig): @@ -1474,8 +1409,7 @@ class ConditionalDetrModel(ConditionalDetrPreTrainedModel): for name, param in self.backbone.conv_encoder.model.named_parameters(): param.requires_grad_(True) - @add_start_docstrings_to_model_forward(CONDITIONAL_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ConditionalDetrModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1489,7 +1423,14 @@ class ConditionalDetrModel(ConditionalDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], ConditionalDetrModelOutput]: r""" - Returns: + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. Examples: @@ -1624,12 +1565,11 @@ class ConditionalDetrMLPPredictionHead(nn.Module): return x -@add_start_docstrings( - """ - CONDITIONAL_DETR Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on +@auto_docstring( + custom_intro=""" + Conditional DETR Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on top, for tasks such as COCO detection. - """, - CONDITIONAL_DETR_START_DOCSTRING, + """ ) class ConditionalDetrForObjectDetection(ConditionalDetrPreTrainedModel): def __init__(self, config: ConditionalDetrConfig): @@ -1657,8 +1597,7 @@ class ConditionalDetrForObjectDetection(ConditionalDetrPreTrainedModel): # as a dict having both a Tensor and a list. return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] - @add_start_docstrings_to_model_forward(CONDITIONAL_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ConditionalDetrObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1673,14 +1612,20 @@ class ConditionalDetrForObjectDetection(ConditionalDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], ConditionalDetrObjectDetectionOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python @@ -1788,13 +1733,11 @@ class ConditionalDetrForObjectDetection(ConditionalDetrPreTrainedModel): ) -@add_start_docstrings( - """ - CONDITIONAL_DETR Model (consisting of a backbone and encoder-decoder Transformer) with a segmentation head on top, +@auto_docstring( + custom_intro=""" + Conditional DETR Model (consisting of a backbone and encoder-decoder Transformer) with a segmentation head on top, for tasks such as COCO panoptic. - - """, - CONDITIONAL_DETR_START_DOCSTRING, + """ ) class ConditionalDetrForSegmentation(ConditionalDetrPreTrainedModel): def __init__(self, config: ConditionalDetrConfig): @@ -1818,8 +1761,7 @@ class ConditionalDetrForSegmentation(ConditionalDetrPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONDITIONAL_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ConditionalDetrSegmentationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1834,6 +1776,14 @@ class ConditionalDetrForSegmentation(ConditionalDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], ConditionalDetrSegmentationOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss, DICE/F-1 loss and Focal loss. List of dicts, each dictionary containing at least the following 3 keys: 'class_labels', 'boxes' and 'masks' (the class labels, @@ -1842,8 +1792,6 @@ class ConditionalDetrForSegmentation(ConditionalDetrPreTrainedModel): `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)` and the masks a `torch.FloatTensor` of shape `(number of bounding boxes in the image, height, width)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/convbert/modeling_convbert.py b/src/transformers/models/convbert/modeling_convbert.py index 486e678b59..0a8f142d1d 100755 --- a/src/transformers/models/convbert/modeling_convbert.py +++ b/src/transformers/models/convbert/modeling_convbert.py @@ -35,15 +35,15 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import ( + auto_docstring, + logging, +) from .configuration_convbert import ConvBertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "YituTech/conv-bert-base" -_CONFIG_FOR_DOC = "ConvBertConfig" - def load_tf_weights_in_convbert(model, config, tf_checkpoint_path): """Load tf checkpoints in a pytorch model.""" @@ -229,12 +229,8 @@ class ConvBertEmbeddings(nn.Module): return embeddings +@auto_docstring class ConvBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ConvBertConfig load_tf_weights = load_tf_weights_in_convbert base_model_prefix = "convbert" @@ -783,74 +779,7 @@ class ConvBertSequenceSummary(nn.Module): return output -CONVBERT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ConvBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CONVBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ConvBERT Model transformer outputting raw hidden-states without any specific head on top.", - CONVBERT_START_DOCSTRING, -) +@auto_docstring class ConvBertModel(ConvBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -878,12 +807,7 @@ class ConvBertModel(ConvBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -965,7 +889,7 @@ class ConvBertGeneratorPredictions(nn.Module): return hidden_states -@add_start_docstrings("""ConvBERT Model with a `language modeling` head on top.""", CONVBERT_START_DOCSTRING) +@auto_docstring class ConvBertForMaskedLM(ConvBertPreTrainedModel): _tied_weights_keys = ["generator.lm_head.weight"] @@ -985,12 +909,7 @@ class ConvBertForMaskedLM(ConvBertPreTrainedModel): def set_output_embeddings(self, word_embeddings): self.generator_lm_head = word_embeddings - @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1070,12 +989,11 @@ class ConvBertClassificationHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ConvBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - CONVBERT_START_DOCSTRING, + """ ) class ConvBertForSequenceClassification(ConvBertPreTrainedModel): def __init__(self, config): @@ -1088,12 +1006,7 @@ class ConvBertForSequenceClassification(ConvBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1165,13 +1078,7 @@ class ConvBertForSequenceClassification(ConvBertPreTrainedModel): ) -@add_start_docstrings( - """ - ConvBERT Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - CONVBERT_START_DOCSTRING, -) +@auto_docstring class ConvBertForMultipleChoice(ConvBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1183,14 +1090,7 @@ class ConvBertForMultipleChoice(ConvBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - CONVBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1205,6 +1105,31 @@ class ConvBertForMultipleChoice(ConvBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1258,13 +1183,7 @@ class ConvBertForMultipleChoice(ConvBertPreTrainedModel): ) -@add_start_docstrings( - """ - ConvBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - CONVBERT_START_DOCSTRING, -) +@auto_docstring class ConvBertForTokenClassification(ConvBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1280,12 +1199,7 @@ class ConvBertForTokenClassification(ConvBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1339,13 +1253,7 @@ class ConvBertForTokenClassification(ConvBertPreTrainedModel): ) -@add_start_docstrings( - """ - ConvBERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - CONVBERT_START_DOCSTRING, -) +@auto_docstring class ConvBertForQuestionAnswering(ConvBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1357,12 +1265,7 @@ class ConvBertForQuestionAnswering(ConvBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1377,16 +1280,6 @@ class ConvBertForQuestionAnswering(ConvBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.convbert( diff --git a/src/transformers/models/convnext/image_processing_convnext_fast.py b/src/transformers/models/convnext/image_processing_convnext_fast.py index 19f959f07d..0d38e7f420 100644 --- a/src/transformers/models/convnext/image_processing_convnext_fast.py +++ b/src/transformers/models/convnext/image_processing_convnext_fast.py @@ -18,8 +18,6 @@ from typing import Dict, List, Optional, Union from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -36,7 +34,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -54,18 +52,16 @@ if is_torchvision_available(): class ConvNextFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + crop_pct (`float`, *optional*): + Percentage of the image to crop. Only has an effect if size < 384. Can be + overridden by `crop_pct` in the`preprocess` method. + """ + crop_pct: Optional[float] -@add_start_docstrings( - r"Constructs a fast ConvNeXT image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - crop_pct (`float`, *optional*): - Percentage of the image to crop. Only has an effect if size < 384. Can be - overridden by `crop_pct` in the`preprocess` method. - """, -) +@auto_docstring class ConvNextImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN @@ -81,14 +77,7 @@ class ConvNextImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[ConvNextFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - crop_pct (`float`, *optional*): - Percentage of the image to crop. Only has an effect if size < 384. Can be - overridden by `crop_pct` in the`preprocess` method. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[ConvNextFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/convnext/modeling_convnext.py b/src/transformers/models/convnext/modeling_convnext.py index d5f65f18b6..0f36abfb8f 100755 --- a/src/transformers/models/convnext/modeling_convnext.py +++ b/src/transformers/models/convnext/modeling_convnext.py @@ -29,30 +29,13 @@ from ...modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_convnext import ConvNextConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "ConvNextConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/convnext-tiny-224" -_EXPECTED_OUTPUT_SHAPE = [1, 768, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/convnext-tiny-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: @@ -268,12 +251,8 @@ class ConvNextEncoder(nn.Module): ) +@auto_docstring class ConvNextPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ConvNextConfig base_model_prefix = "convnext" main_input_name = "pixel_values" @@ -295,35 +274,7 @@ class ConvNextPreTrainedModel(PreTrainedModel): module.layer_scale_parameter.data.fill_(self.config.layer_scale_init_value) -CONVNEXT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ConvNextConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CONVNEXT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`ConvNextImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ConvNext model outputting raw features without any specific head on top.", - CONVNEXT_START_DOCSTRING, -) +@auto_docstring class ConvNextModel(ConvNextPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -338,14 +289,7 @@ class ConvNextModel(ConvNextPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVNEXT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -383,12 +327,11 @@ class ConvNextModel(ConvNextPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ConvNext Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - CONVNEXT_START_DOCSTRING, + """ ) class ConvNextForImageClassification(ConvNextPreTrainedModel): def __init__(self, config): @@ -405,13 +348,7 @@ class ConvNextForImageClassification(ConvNextPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVNEXT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -466,11 +403,10 @@ class ConvNextForImageClassification(ConvNextPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ConvNeXt backbone, to be used with frameworks like DETR and MaskFormer. - """, - CONVNEXT_START_DOCSTRING, + """ ) class ConvNextBackbone(ConvNextPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -490,17 +426,14 @@ class ConvNextBackbone(ConvNextPreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVNEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/convnextv2/modeling_convnextv2.py b/src/transformers/models/convnextv2/modeling_convnextv2.py index a3af36bac3..9fd15612fc 100644 --- a/src/transformers/models/convnextv2/modeling_convnextv2.py +++ b/src/transformers/models/convnextv2/modeling_convnextv2.py @@ -29,30 +29,13 @@ from ...modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_convnextv2 import ConvNextV2Config logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "ConvNextV2Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/convnextv2-tiny-1k-224" -_EXPECTED_OUTPUT_SHAPE = [1, 768, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/convnextv2-tiny-1k-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: @@ -288,12 +271,8 @@ class ConvNextV2Encoder(nn.Module): ) +@auto_docstring class ConvNextV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ConvNextV2Config base_model_prefix = "convnextv2" main_input_name = "pixel_values" @@ -315,34 +294,7 @@ class ConvNextV2PreTrainedModel(PreTrainedModel): module.bias.data.zero_() -CONVNEXTV2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ConvNextV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CONVNEXTV2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`ConvNextImageProcessor`]. See - [`ConvNextImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ConvNextV2 model outputting raw features without any specific head on top.", - CONVNEXTV2_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.convnext.modeling_convnext.ConvNextModel with CONVNEXT->CONVNEXTV2, ConvNext->ConvNextV2 class ConvNextV2Model(ConvNextV2PreTrainedModel): def __init__(self, config): @@ -358,14 +310,7 @@ class ConvNextV2Model(ConvNextV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -403,12 +348,11 @@ class ConvNextV2Model(ConvNextV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ConvNextV2 Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - CONVNEXTV2_START_DOCSTRING, + """ ) # Copied from transformers.models.convnext.modeling_convnext.ConvNextForImageClassification with CONVNEXT->CONVNEXTV2,ConvNext->ConvNextV2,convnext->convnextv2 class ConvNextV2ForImageClassification(ConvNextV2PreTrainedModel): @@ -426,13 +370,7 @@ class ConvNextV2ForImageClassification(ConvNextV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -487,11 +425,10 @@ class ConvNextV2ForImageClassification(ConvNextV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ConvNeXT V2 backbone, to be used with frameworks like DETR and MaskFormer. - """, - CONVNEXTV2_START_DOCSTRING, + """ ) # Copied from transformers.models.convnext.modeling_convnext.ConvNextBackbone with CONVNEXT->CONVNEXTV2,ConvNext->ConvNextV2,facebook/convnext-tiny-224->facebook/convnextv2-tiny-1k-224 class ConvNextV2Backbone(ConvNextV2PreTrainedModel, BackboneMixin): @@ -512,17 +449,14 @@ class ConvNextV2Backbone(ConvNextV2PreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/cpmant/modeling_cpmant.py b/src/transformers/models/cpmant/modeling_cpmant.py index a587a11f87..e437672aa5 100755 --- a/src/transformers/models/cpmant/modeling_cpmant.py +++ b/src/transformers/models/cpmant/modeling_cpmant.py @@ -27,15 +27,12 @@ from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_cpmant import CpmAntConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "openbmb/cpm-ant-10b" -_CONFIG_FOR_DOC = "CpmAntConfig" - class CpmAntLayerNorm(nn.Module): """ @@ -523,12 +520,8 @@ class CpmAntOutput(nn.Module): return hidden_states +@auto_docstring class CpmAntPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CpmAntConfig base_model_prefix = "cpmant" @@ -551,45 +544,7 @@ class CpmAntPreTrainedModel(PreTrainedModel): module.relative_attention_bias.data.normal_(mean=0.0, std=self.config.init_std) -CPMANT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters - config ([`~CpmAntConfig`]): Model configuration class with all the parameters of the - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CPMANT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare CPMAnt Model outputting raw hidden-states without any specific head on top.", - CPMANT_START_DOCSTRING, -) +@auto_docstring class CpmAntModel(CpmAntPreTrainedModel): def __init__(self, config: CpmAntConfig): super().__init__(config) @@ -628,12 +583,7 @@ class CpmAntModel(CpmAntPreTrainedModel): attention_mask = mask_1d.view(batch, seqlen, 1) & mask_1d.view(batch, 1, seqlen) & attention_mask return attention_mask - @add_start_docstrings_to_model_forward(CPMANT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -644,6 +594,15 @@ class CpmAntModel(CpmAntPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]: + r""" + input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -731,11 +690,10 @@ class CpmAntModel(CpmAntPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The CPMAnt Model with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - CPMANT_START_DOCSTRING, + """ ) class CpmAntForCausalLM(CpmAntPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -750,12 +708,7 @@ class CpmAntForCausalLM(CpmAntPreTrainedModel, GenerationMixin): ) self.post_init() - @add_start_docstrings_to_model_forward(CPMANT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -769,31 +722,15 @@ class CpmAntForCausalLM(CpmAntPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - Args: - input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`): - Indices of input sequence tokens in the vocabulary. + input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`): + Indices of input sequence tokens in the vocabulary. - Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. + Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. - [What are input IDs?](../glossary#input-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. - labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - CPMAnt will process attention mask automatically, this parameter is a dummy parameter for - text-generation pipeline. + [What are input IDs?](../glossary#input-ids) + labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Example: diff --git a/src/transformers/models/csm/modeling_csm.py b/src/transformers/models/csm/modeling_csm.py index 03cbc07df4..58042c64ab 100644 --- a/src/transformers/models/csm/modeling_csm.py +++ b/src/transformers/models/csm/modeling_csm.py @@ -36,16 +36,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, ModelOutput, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from ..auto import AutoModel from .configuration_csm import CsmConfig, CsmDepthDecoderConfig from .generation_csm import CsmGenerationMixin @@ -58,7 +49,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "CsmConfig" @dataclass @@ -120,30 +110,12 @@ class CsmOutputWithPast(ModelOutput): backbone_loss: Optional[torch.FloatTensor] = None -START_DOCSTRING_BASE = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`{config_class}`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -CSM_START_DOCSTRING = r"""{}""".format(START_DOCSTRING_BASE.format(config_class="CsmConfig")) - - -@add_start_docstrings( - "The bare Csm Model outputting raw hidden-states without any specific head on top.", - CSM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare Csm Model outputting raw hidden-states without any specific head on top. + """ ) +@auto_docstring class CsmPreTrainedModel(PreTrainedModel): config_class = CsmConfig base_model_prefix = "model" @@ -450,104 +422,8 @@ class CsmDecoderLayer(GradientCheckpointingLayer): return outputs -CSM_DEPTH_DECODER_START_DOCSTRING = r"""{}""".format(START_DOCSTRING_BASE.format(config_class="CsmDepthDecoderConfig")) - - -INPUTS_DOCSTRING_BASE = r""" - Args: - {input_ids_docstring} - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -DEPTH_DECODER_INPUT_IDS_DOCSTRING = r"""input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids)""" - - -CSM_DEPTH_DECODER_INPUTS_DOCSTRING = r"""{}""".format( - INPUTS_DOCSTRING_BASE.format(input_ids_docstring=DEPTH_DECODER_INPUT_IDS_DOCSTRING) -) - - -@add_start_docstrings( - "The bare CsmDepthDecoderModel outputting raw hidden-states without any specific head on top.", - CSM_DEPTH_DECODER_START_DOCSTRING, -) +@auto_docstring class CsmDepthDecoderModel(CsmPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`CsmDecoderLayer`] - - Args: - config: CsmDepthDecoderConfig - """ - config_class = CsmDepthDecoderConfig def __init__(self, config): @@ -573,7 +449,7 @@ class CsmDepthDecoderModel(CsmPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_DEPTH_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -838,13 +714,12 @@ class CsmCodebooksHead(nn.Module): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The CsmDepthDecoder Model transformer, with a [`CsmCodebooksHead`] on top, which can be seen a position-specific language modeling head, allowing to use a different linear layer for each codebook (e.g. position 0 is the first codebook and uses the first codebook head, etc.) - """, - CSM_DEPTH_DECODER_START_DOCSTRING, + """ ) class CsmDepthDecoderForCausalLM(CsmPreTrainedModel, GenerationMixin): _tied_weights_keys = None @@ -873,8 +748,7 @@ class CsmDepthDecoderForCausalLM(CsmPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_DEPTH_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -892,43 +766,13 @@ class CsmDepthDecoderForCausalLM(CsmPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: - - Example: - - ```python - >>> from transformers import AutoTokenizer, CsmDepthDecoderForCausalLM - - >>> model = CsmDepthDecoderForCausalLM.from_pretrained("meta-csm_depth_decoder/CsmDepthDecoder-2-7b-hf") - >>> tokenizer = AutoTokenizer.from_pretrained("meta-csm_depth_decoder/CsmDepthDecoder-2-7b-hf") - - >>> prompt = "Hey, are you conscious? Can you talk to me?" - >>> inputs = tokenizer(prompt, return_tensors="pt") - - >>> # Generate - >>> generate_ids = model.generate(inputs.input_ids, max_length=30) - >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." - ``` - backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*): - The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model) - is provided in the `input_ids` argument. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*): + The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model) + is provided in the `input_ids` argument. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1018,33 +862,8 @@ class CsmBackboneModelEmbeddings(nn.Module): return input_embeds -INPUT_IDS_DOCSTRING = r"""input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): - 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input - requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. - - 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids)""" - - -CSM_BACKBONE_INPUTS_DOCSTRING = r"""{}""".format(INPUTS_DOCSTRING_BASE.format(input_ids_docstring=INPUT_IDS_DOCSTRING)) - - -@add_start_docstrings( - "The bare CsmBackboneModel Model outputting raw hidden-states without any specific head on top.", - CSM_START_DOCSTRING, -) +@auto_docstring class CsmBackboneModel(CsmPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`CsmDecoderLayer`] - - Args: - config: CsmBackboneModelConfig - """ - def __init__(self, config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1067,7 +886,7 @@ class CsmBackboneModel(CsmPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_BACKBONE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1081,6 +900,18 @@ class CsmBackboneModel(CsmPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **flash_attn_kwargs: Unpack[FlashAttentionKwargs], ) -> BaseModelOutputWithPast: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): + 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input + requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. + + 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1286,14 +1117,10 @@ class CsmBackboneModel(CsmPreTrainedModel): return causal_mask -CSM_INPUTS_DOCSTRING = r"""{}""".format(INPUTS_DOCSTRING_BASE.format(input_ids_docstring=INPUT_IDS_DOCSTRING)) - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Csm model consists of two llama-like auto-regressive transformer models: a backbone model that predicts the first codebook token and a depth decoder that predicts the other codebook tokens. - """, - CSM_START_DOCSTRING, + """ ) class CsmForConditionalGeneration(CsmPreTrainedModel, CsmGenerationMixin): _tied_weights_keys = [ @@ -1481,8 +1308,7 @@ class CsmForConditionalGeneration(CsmPreTrainedModel, CsmGenerationMixin): return model_inputs @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CsmOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1501,27 +1327,33 @@ class CsmForConditionalGeneration(CsmPreTrainedModel, CsmGenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CsmOutputWithPast]: r""" - input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`, *optional*): - Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input. - If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences - where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2, - the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]]. + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): + 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input + requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[config.audio_token_id, -100, -101]`. - Requires targeted `input_values` to be provided as audio tokens will be infered from it using the `codec_model`. - - `config.audio_token_id` indicates an audio frames (considering sequence length elements as frames) - - `-100` will be ignored in the loss computation - - `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels) + 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. - Such labels can be prepared using `output_labels=True` when calling [`CsmProcessor`]. + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. - logits_to_keep (`int` or `torch.Tensor`, *optional*): - Kept for compatibility. Does not support another value than: - 1. `0`, which is equivalent to keeping all logits, used in the training regime - 2. `1`, which is equivalent to keeping only the last logit, used in the generation regime + [What are input IDs?](../glossary#input-ids) + input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`, *optional*): + Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input. + If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences + where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2, + the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]]. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[config.audio_token_id, -100, -101]`. + Requires targeted `input_values` to be provided as audio tokens will be infered from it using the `codec_model`. + - `config.audio_token_id` indicates an audio frames (considering sequence length elements as frames) + - `-100` will be ignored in the loss computation + - `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels) - Returns: + Such labels can be prepared using `output_labels=True` when calling [`CsmProcessor`]. + logits_to_keep (`int` or `torch.Tensor`, *optional*): + Kept for compatibility. Does not support another value than: + 1. `0`, which is equivalent to keeping all logits, used in the training regime + 2. `1`, which is equivalent to keeping only the last logit, used in the generation regime Example: diff --git a/src/transformers/models/csm/modular_csm.py b/src/transformers/models/csm/modular_csm.py index ed3d571034..8b7a54d276 100644 --- a/src/transformers/models/csm/modular_csm.py +++ b/src/transformers/models/csm/modular_csm.py @@ -22,20 +22,10 @@ import torch.nn as nn from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - BaseModelOutputWithPast, - CausalLMOutputWithPast, -) +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging from ..auto import AutoModel from ..llama.modeling_llama import ( KwargsForCausalLM, @@ -47,15 +37,11 @@ from ..llama.modeling_llama import ( LlamaRMSNorm, LlamaRotaryEmbedding, ) -from .configuration_csm import ( - CsmConfig, - CsmDepthDecoderConfig, -) +from .configuration_csm import CsmConfig, CsmDepthDecoderConfig from .generation_csm import CsmGenerationMixin logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "CsmConfig" @dataclass @@ -117,33 +103,12 @@ class CsmOutputWithPast(ModelOutput): backbone_loss: Optional[torch.FloatTensor] = None -START_DOCSTRING_BASE = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`{config_class}`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -CSM_DEPTH_DECODER_START_DOCSTRING = r"""{}""".format(START_DOCSTRING_BASE.format(config_class="CsmDepthDecoderConfig")) - - -CSM_START_DOCSTRING = r"""{}""".format(START_DOCSTRING_BASE.format(config_class="CsmConfig")) - - -@add_start_docstrings( - "The bare Csm Model outputting raw hidden-states without any specific head on top.", - CSM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare Csm Model outputting raw hidden-states without any specific head on top. + """ ) +@auto_docstring class CsmPreTrainedModel(PreTrainedModel): config_class = CsmConfig base_model_prefix = "model" @@ -177,104 +142,6 @@ class CsmPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -INPUTS_DOCSTRING_BASE = r""" - Args: - {input_ids_docstring} - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -DEPTH_DECODER_INPUT_IDS_DOCSTRING = r"""input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids)""" - - -INPUT_IDS_DOCSTRING = r"""input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): - 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input - requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. - - 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids)""" - - -CSM_DEPTH_DECODER_INPUTS_DOCSTRING = r"""{}""".format( - INPUTS_DOCSTRING_BASE.format(input_ids_docstring=DEPTH_DECODER_INPUT_IDS_DOCSTRING) -) - - -CSM_BACKBONE_INPUTS_DOCSTRING = r"""{}""".format(INPUTS_DOCSTRING_BASE.format(input_ids_docstring=INPUT_IDS_DOCSTRING)) - - # manually specify names for correct naming when converting from modualr class CsmRMSNorm(LlamaRMSNorm): pass @@ -296,18 +163,8 @@ class CsmDecoderLayer(LlamaDecoderLayer): pass -@add_start_docstrings( - "The bare CsmDepthDecoderModel outputting raw hidden-states without any specific head on top.", - CSM_DEPTH_DECODER_START_DOCSTRING, -) +@auto_docstring class CsmDepthDecoderModel(LlamaModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`CsmDecoderLayer`] - - Args: - config: CsmDepthDecoderConfig - """ - config_class = CsmDepthDecoderConfig def __init__(self, config): @@ -316,7 +173,7 @@ class CsmDepthDecoderModel(LlamaModel): self.inputs_embeds_projector = nn.Linear(config.backbone_hidden_size, config.hidden_size, bias=False) @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_DEPTH_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -455,13 +312,12 @@ class CsmCodebooksHead(nn.Module): return hidden_states -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The CsmDepthDecoder Model transformer, with a [`CsmCodebooksHead`] on top, which can be seen a position-specific language modeling head, allowing to use a different linear layer for each codebook (e.g. position 0 is the first codebook and uses the first codebook head, etc.) - """, - CSM_DEPTH_DECODER_START_DOCSTRING, + """ ) class CsmDepthDecoderForCausalLM(LlamaForCausalLM, GenerationMixin): _tied_weights_keys = None @@ -503,8 +359,7 @@ class CsmDepthDecoderForCausalLM(LlamaForCausalLM, GenerationMixin): return model_inputs @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_DEPTH_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -596,36 +451,34 @@ class CsmBackboneModelEmbeddings(nn.Module): return input_embeds -@add_start_docstrings( - "The bare CsmBackboneModel Model outputting raw hidden-states without any specific head on top.", - CSM_START_DOCSTRING, -) +@auto_docstring class CsmBackboneModel(LlamaModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`CsmDecoderLayer`] - - Args: - config: CsmBackboneModelConfig - """ - def __init__(self, config): super().__init__(config) self.embed_tokens = CsmBackboneModelEmbeddings(config) @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_BACKBONE_INPUTS_DOCSTRING) + @auto_docstring def forward(self, **super_kwargs): + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): + 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input + requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. + + 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ return super().forward(**super_kwargs) -CSM_INPUTS_DOCSTRING = r"""{}""".format(INPUTS_DOCSTRING_BASE.format(input_ids_docstring=INPUT_IDS_DOCSTRING)) - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Csm model consists of two llama-like auto-regressive transformer models: a backbone model that predicts the first codebook token and a depth decoder that predicts the other codebook tokens. - """, - CSM_START_DOCSTRING, + """ ) class CsmForConditionalGeneration(CsmPreTrainedModel, CsmGenerationMixin): _tied_weights_keys = [ @@ -813,8 +666,7 @@ class CsmForConditionalGeneration(CsmPreTrainedModel, CsmGenerationMixin): return model_inputs @can_return_tuple - @add_start_docstrings_to_model_forward(CSM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CsmOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -833,27 +685,33 @@ class CsmForConditionalGeneration(CsmPreTrainedModel, CsmGenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CsmOutputWithPast]: r""" - input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`, *optional*): - Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input. - If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences - where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2, - the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]]. + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): + 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input + requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[config.audio_token_id, -100, -101]`. - Requires targeted `input_values` to be provided as audio tokens will be infered from it using the `codec_model`. - - `config.audio_token_id` indicates an audio frames (considering sequence length elements as frames) - - `-100` will be ignored in the loss computation - - `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels) + 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. - Such labels can be prepared using `output_labels=True` when calling [`CsmProcessor`]. + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. - logits_to_keep (`int` or `torch.Tensor`, *optional*): - Kept for compatibility. Does not support another value than: - 1. `0`, which is equivalent to keeping all logits, used in the training regime - 2. `1`, which is equivalent to keeping only the last logit, used in the generation regime + [What are input IDs?](../glossary#input-ids) + input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`, *optional*): + Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input. + If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences + where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2, + the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]]. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[config.audio_token_id, -100, -101]`. + Requires targeted `input_values` to be provided as audio tokens will be infered from it using the `codec_model`. + - `config.audio_token_id` indicates an audio frames (considering sequence length elements as frames) + - `-100` will be ignored in the loss computation + - `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels) - Returns: + Such labels can be prepared using `output_labels=True` when calling [`CsmProcessor`]. + logits_to_keep (`int` or `torch.Tensor`, *optional*): + Kept for compatibility. Does not support another value than: + 1. `0`, which is equivalent to keeping all logits, used in the training regime + 2. `1`, which is equivalent to keeping only the last logit, used in the generation regime Example: diff --git a/src/transformers/models/ctrl/modeling_ctrl.py b/src/transformers/models/ctrl/modeling_ctrl.py index 5426789c04..1896d6ea41 100644 --- a/src/transformers/models/ctrl/modeling_ctrl.py +++ b/src/transformers/models/ctrl/modeling_ctrl.py @@ -26,14 +26,15 @@ from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import Conv1D, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import ( + auto_docstring, + logging, +) from .configuration_ctrl import CTRLConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "CTRLConfig" - def angle_defn(pos, i, d_model_size): angle_rates = 1 / torch.pow(10000, (2 * (i // 2)) / d_model_size) @@ -205,12 +206,8 @@ class EncoderLayer(nn.Module): return outputs +@auto_docstring class CTRLPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CTRLConfig base_model_prefix = "transformer" @@ -231,87 +228,7 @@ class CTRLPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -CTRL_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`CTRLConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CTRL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` - (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. - - If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and - [`PreTrainedTokenizer.encode`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Tuple[Tuple[torch.FloatTensor]]` of length `config.n_layers`): - Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see - `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have - their past given to this model should not be passed as input ids as they have already been computed. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, input_ids_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.", - CTRL_START_DOCSTRING, -) +@auto_docstring class CTRLModel(CTRLPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -345,8 +262,7 @@ class CTRLModel(CTRLPreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].multi_head_attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -363,7 +279,17 @@ class CTRLModel(CTRLPreTrainedModel): **kwargs, # NOOP kwargs, for now ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + [`PreTrainedTokenizer.encode`] for details. + + [What are input IDs?](../glossary#input-ids) Example: @@ -498,12 +424,11 @@ class CTRLModel(CTRLPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - CTRL_START_DOCSTRING, + """ ) class CTRLLMHeadModel(CTRLPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -522,8 +447,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -541,13 +465,22 @@ class CTRLLMHeadModel(CTRLPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + [`PreTrainedTokenizer.encode`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Example: ```python @@ -647,8 +580,8 @@ class CTRLLMHeadModel(CTRLPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The CTRL Model transformer with a sequence classification head on top (linear layer). [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last @@ -656,8 +589,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel, GenerationMixin): each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - CTRL_START_DOCSTRING, + """ ) class CTRLForSequenceClassification(CTRLPreTrainedModel): def __init__(self, config): @@ -669,8 +601,7 @@ class CTRLForSequenceClassification(CTRLPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -687,13 +618,22 @@ class CTRLForSequenceClassification(CTRLPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and + [`PreTrainedTokenizer.encode`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Example of single-label classification: ```python diff --git a/src/transformers/models/cvt/modeling_cvt.py b/src/transformers/models/cvt/modeling_cvt.py index 0088b8b440..eb9294dd43 100644 --- a/src/transformers/models/cvt/modeling_cvt.py +++ b/src/transformers/models/cvt/modeling_cvt.py @@ -23,26 +23,14 @@ import torch.utils.checkpoint from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss -from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...modeling_outputs import ImageClassifierOutputWithNoAttention, ModelOutput from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import logging +from ...utils import auto_docstring, logging from .configuration_cvt import CvtConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "CvtConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/cvt-13" -_EXPECTED_OUTPUT_SHAPE = [1, 384, 14, 14] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "microsoft/cvt-13" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - @dataclass class BaseModelOutputWithCLSToken(ModelOutput): @@ -523,12 +511,8 @@ class CvtEncoder(nn.Module): ) +@auto_docstring class CvtPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = CvtConfig base_model_prefix = "cvt" main_input_name = "pixel_values" @@ -550,36 +534,13 @@ class CvtPreTrainedModel(PreTrainedModel): ) -CVT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`CvtConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -CVT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CvtImageProcessor.__call__`] - for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Cvt Model transformer outputting raw hidden-states without any specific head on top.", - CVT_START_DOCSTRING, -) +@auto_docstring class CvtModel(CvtPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config self.encoder = CvtEncoder(config) @@ -593,14 +554,7 @@ class CvtModel(CvtPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(CVT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithCLSToken, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -632,12 +586,11 @@ class CvtModel(CvtPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Cvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - CVT_START_DOCSTRING, + """ ) class CvtForImageClassification(CvtPreTrainedModel): def __init__(self, config): @@ -654,13 +607,7 @@ class CvtForImageClassification(CvtPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(CVT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/d_fine/modeling_d_fine.py b/src/transformers/models/d_fine/modeling_d_fine.py index 784061a9ec..1db385b52d 100644 --- a/src/transformers/models/d_fine/modeling_d_fine.py +++ b/src/transformers/models/d_fine/modeling_d_fine.py @@ -32,21 +32,11 @@ from ...image_transforms import center_to_corners_format, corners_to_center_form from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torchdynamo_compiling, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, is_torchdynamo_compiling, torch_int from ...utils.backbone_utils import load_backbone from .configuration_d_fine import DFineConfig -_CONFIG_FOR_DOC = "DFineConfig" - - def multi_scale_deformable_attention_v2( value: Tensor, value_spatial_shapes: Tensor, @@ -922,64 +912,11 @@ def get_contrastive_denoising_training_group( return input_query_class, input_query_bbox, attn_mask, denoising_meta_values -DFine_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DFineConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DFine_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`DFineImageProcessor.__call__`] for details. - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - labels (`List[Dict]` of len `(batch_size,)`, *optional*): - Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the - following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch - respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes - in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - def _get_clones(partial_module, N): return nn.ModuleList([partial_module() for i in range(N)]) +@auto_docstring class DFinePreTrainedModel(PreTrainedModel): config_class = DFineConfig base_model_prefix = "d_fine" @@ -1366,11 +1303,10 @@ class DFineDecoder(DFinePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RT-DETR Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. - """, - DFine_START_DOCSTRING, + """ ) class DFineModel(DFinePreTrainedModel): def __init__(self, config: DFineConfig): @@ -1495,8 +1431,7 @@ class DFineModel(DFinePreTrainedModel): return anchors, valid_mask - @add_start_docstrings_to_model_forward(DFine_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DFineModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1510,7 +1445,17 @@ class DFineModel(DFinePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DFineModelOutput]: r""" - Returns: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`List[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. Examples: @@ -1709,12 +1654,11 @@ class DFineModel(DFinePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RT-DETR Model (consisting of a backbone and encoder-decoder) outputting bounding boxes and logits to be further decoded into scores and classes. - """, - DFine_START_DOCSTRING, + """ ) class DFineForObjectDetection(DFinePreTrainedModel): # When using clones, all layers > 0 will be clones, but layer 0 *is* required @@ -1756,8 +1700,7 @@ class DFineForObjectDetection(DFinePreTrainedModel): # as a dict having both a Tensor and a list. return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class, outputs_coord)] - @add_start_docstrings_to_model_forward(DFine_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DFineObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1772,14 +1715,18 @@ class DFineForObjectDetection(DFinePreTrainedModel): **loss_kwargs, ) -> Union[Tuple[torch.FloatTensor], DFineObjectDetectionOutput]: r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/dab_detr/modeling_dab_detr.py b/src/transformers/models/dab_detr/modeling_dab_detr.py index 3a60a07edf..03c513f0d5 100644 --- a/src/transformers/models/dab_detr/modeling_dab_detr.py +++ b/src/transformers/models/dab_detr/modeling_dab_detr.py @@ -27,10 +27,8 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttenti from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from ...utils.backbone_utils import load_backbone from .configuration_dab_detr import DabDetrConfig @@ -38,9 +36,6 @@ from .configuration_dab_detr import DabDetrConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DabDetrConfig" -_CHECKPOINT_FOR_DOC = "IDEA-Research/dab_detr-base" - @dataclass # Copied from transformers.models.conditional_detr.modeling_conditional_detr.ConditionalDetrDecoderOutput with ConditionalDetr->DabDetr,Conditional DETR->DAB-DETR,2 (anchor points)->4 (anchor points) @@ -859,6 +854,7 @@ class DabDetrMLP(nn.Module): # Modified from transformers.models.detr.modeling_detr.DetrPreTrainedModel with Detr->DabDetr +@auto_docstring class DabDetrPreTrainedModel(PreTrainedModel): config_class = DabDetrConfig base_model_prefix = "model" @@ -894,61 +890,6 @@ class DabDetrPreTrainedModel(PreTrainedModel): module.class_embed.bias.data.fill_(bias_value) -DAB_DETR_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DabDetrConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DAB_DETR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`AutoImageProcessor`]. See [`DetrImageProcessor.__call__`] - for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): - Not used by default. Can be used to mask object queries. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Modified from transformers.models.detr.modeling_detr.DetrEncoder with Detr->DabDetr,DETR->ConditionalDETR class DabDetrEncoder(DabDetrPreTrainedModel): """ @@ -1273,12 +1214,11 @@ class DabDetrDecoder(DabDetrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The bare DAB-DETR Model (consisting of a backbone and encoder-decoder Transformer) outputting raw hidden-states, intermediate hidden states, reference points, output coordinates without any specific head on top. - """, - DAB_DETR_START_DOCSTRING, + """ ) class DabDetrModel(DabDetrPreTrainedModel): def __init__(self, config: DabDetrConfig): @@ -1338,8 +1278,7 @@ class DabDetrModel(DabDetrPreTrainedModel): for name, param in self.backbone.conv_encoder.model.named_parameters(): param.requires_grad_(True) - @add_start_docstrings_to_model_forward(DAB_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DabDetrModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1353,7 +1292,14 @@ class DabDetrModel(DabDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DabDetrModelOutput]: r""" - Returns: + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. Examples: @@ -1543,12 +1489,11 @@ class DabDetrMHAttentionMap(nn.Module): return weights -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DAB_DETR Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on top, for tasks such as COCO detection. - """, - DAB_DETR_START_DOCSTRING, + """ ) class DabDetrForObjectDetection(DabDetrPreTrainedModel): # When using clones, all layers > 0 will be clones, but layer 0 *is* required @@ -1587,8 +1532,7 @@ class DabDetrForObjectDetection(DabDetrPreTrainedModel): # as a dict having both a Tensor and a list. return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] - @add_start_docstrings_to_model_forward(DAB_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DabDetrObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1603,14 +1547,20 @@ class DabDetrForObjectDetection(DabDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DabDetrObjectDetectionOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/dac/modeling_dac.py b/src/transformers/models/dac/modeling_dac.py index 47a45f44e9..519a4bcd44 100644 --- a/src/transformers/models/dac/modeling_dac.py +++ b/src/transformers/models/dac/modeling_dac.py @@ -24,19 +24,10 @@ import torch.nn as nn import torch.nn.functional as F from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring from .configuration_dac import DacConfig -# General docstring -_CONFIG_FOR_DOC = "DacConfig" - - @dataclass class DacOutput(ModelOutput): """ @@ -479,11 +470,8 @@ class DacEncoder(nn.Module): return hidden_state +@auto_docstring class DacPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. - """ - config_class = DacConfig base_model_prefix = "dac" main_input_name = "input_values" @@ -556,36 +544,10 @@ class DacPreTrainedModel(PreTrainedModel): nn.utils.remove_weight_norm(layer.res_unit3.conv2) -DAC_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DacConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DAC_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.Tensor` of shape `(batch_size, 1, time_steps)`). - Audio data to encode, - n_quantizers (`int`, *optional*): - Number of quantizers to use. If `None`, all quantizers are used. Default is `None`. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The DAC (Descript Audio Codec) model.", - DAC_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The DAC (Descript Audio Codec) model. + """ ) class DacModel(DacPreTrainedModel): def __init__(self, config: DacConfig): @@ -604,25 +566,18 @@ class DacModel(DacPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @replace_return_docstrings(output_type=DacEncoderOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def encode( self, input_values: torch.Tensor, n_quantizers: Optional[int] = None, return_dict: Optional[bool] = None, ): - """ - Encode given audio data and return quantized latent codes - - Args: - input_values (`torch.Tensor of shape `(batch_size, 1, time_steps)`): - Input audio data to encode, - n_quantizers (int, *optional*): - Number of quantizers to use. If None, all quantizers are used. Default is None. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - Returns: - + r""" + input_values (`torch.Tensor of shape `(batch_size, 1, time_steps)`): + Input audio data to encode, + n_quantizers (int, *optional*): + Number of quantizers to use. If None, all quantizers are used. Default is None. """ return_dict = return_dict if return_dict is not None else self.config.return_dict @@ -638,27 +593,20 @@ class DacModel(DacPreTrainedModel): return DacEncoderOutput(loss, quantized_representation, audio_codes, projected_latents) - @replace_return_docstrings(output_type=DacDecoderOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def decode( self, quantized_representation: Optional[torch.Tensor] = None, audio_codes: Optional[torch.Tensor] = None, return_dict: Optional[bool] = None, ): - """Decode given latent codes and return audio data - - Args: - quantized_representation (torch.Tensor of shape `(batch_size, dimension, time_steps)`, *optional*): - Quantized continuous representation of input. - audio_codes (`torch.Tensor` of shape `(batch_size, num_codebooks, time_steps)`, *optional*): - The codebook indices for each codebook, representing the quantized discrete - representation of the input. This parameter should be provided if you want - to decode directly from the audio codes (it will overwrite quantized_representation). - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: - + r""" + quantized_representation (torch.Tensor of shape `(batch_size, dimension, time_steps)`, *optional*): + Quantized continuous representation of input. + audio_codes (`torch.Tensor` of shape `(batch_size, num_codebooks, time_steps)`, *optional*): + The codebook indices for each codebook, representing the quantized discrete + representation of the input. This parameter should be provided if you want + to decode directly from the audio codes (it will overwrite quantized_representation). """ if quantized_representation is None and audio_codes is None: @@ -676,16 +624,19 @@ class DacModel(DacPreTrainedModel): return DacDecoderOutput(audio_values) - @add_start_docstrings_to_model_forward(DAC_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DacOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: torch.Tensor, n_quantizers: Optional[int] = None, return_dict: Optional[bool] = None, ): - """ - Returns: + r""" + input_values (`torch.Tensor` of shape `(batch_size, 1, time_steps)`): + Audio data to encode. + n_quantizers (`int`, *optional*): + Number of quantizers to use. If `None`, all quantizers are used. Default is `None`. + Examples: ```python diff --git a/src/transformers/models/data2vec/modeling_data2vec_audio.py b/src/transformers/models/data2vec/modeling_data2vec_audio.py index 7f799e8129..3db2fc0c68 100755 --- a/src/transformers/models/data2vec/modeling_data2vec_audio.py +++ b/src/transformers/models/data2vec/modeling_data2vec_audio.py @@ -26,13 +26,7 @@ from ...modeling_outputs import ( XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_peft_available, - logging, -) +from ...utils import auto_docstring, is_peft_available, logging from .configuration_data2vec_audio import Data2VecAudioConfig @@ -42,12 +36,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/data2vec-audio-base-960h" - -# General docstring -_CONFIG_FOR_DOC = "Data2VecAudioConfig" - class Data2VecAudioConvLayer(nn.Module): def __init__(self, config, layer_id=0): @@ -769,12 +757,8 @@ class Data2VecAudioAdapter(nn.Module): return hidden_states +@auto_docstring class Data2VecAudioPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Data2VecAudioConfig base_model_prefix = "data2vec_audio" main_input_name = "input_values" @@ -970,70 +954,10 @@ def _compute_mask_indices( return spec_aug_mask -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - - -DATA2VEC_AUDIO_START_DOCSTRING = r""" - Data2VecAudio was proposed in [data2vec: A General Framework for Self-supervised Learning in Speech, Vision and - Language](https://arxiv.org/pdf/2202.03555) by Alexei Baevski, Wei-Ning Hsu, Qiantong Xu, Arun Babu, Jiatao Gu and - Michael Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Data2VecAudioConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DATA2VEC_AUDIO_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should be passed if the corresponding processor has `config.return_attention_mask == - True`, which is the case for all pre-trained Data2Vec Audio models. Be aware that that even with - `attention_mask`, zero-padded inputs will have slightly different outputs compared to non-padded inputs - because there are more than one convolutional layer in the positional encodings. For a more detailed - explanation, see [here](https://github.com/huggingface/transformers/issues/25621#issuecomment-1713759349). - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - Data2VecAudioBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare Data2VecAudio Model transformer outputting raw hidden-states without any specific head on top.", - DATA2VEC_AUDIO_START_DOCSTRING, -) +@auto_docstring class Data2VecAudioModel(Data2VecAudioPreTrainedModel): def __init__(self, config: Data2VecAudioConfig): super().__init__(config) @@ -1105,14 +1029,7 @@ class Data2VecAudioModel(Data2VecAudioPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Data2VecAudioBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1122,6 +1039,11 @@ class Data2VecAudioModel(Data2VecAudioPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Data2VecAudioBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1168,17 +1090,20 @@ class Data2VecAudioModel(Data2VecAudioPreTrainedModel): _HIDDEN_STATES_START_POSITION = 2 -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 66.95 - -@add_start_docstrings( - """Data2VecAudio Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - DATA2VEC_AUDIO_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Data2VecAudio Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel): def __init__(self, config): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`Data2VecAudioForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.data2vec_audio = Data2VecAudioModel(config) @@ -1218,14 +1143,7 @@ class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel): """ self.data2vec_audio.feature_extractor._freeze_parameters() - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1297,12 +1215,11 @@ class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Data2VecAudio Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like + SUPERB Keyword Spotting. """ - Data2VecAudio Model with a sequence classification head on top (a linear layer over the pooled output) for tasks - like SUPERB Keyword Spotting. - """, - DATA2VEC_AUDIO_START_DOCSTRING, ) class Data2VecAudioForSequenceClassification(Data2VecAudioPreTrainedModel): def __init__(self, config): @@ -1349,13 +1266,7 @@ class Data2VecAudioForSequenceClassification(Data2VecAudioPreTrainedModel): for param in self.data2vec_audio.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1366,6 +1277,11 @@ class Data2VecAudioForSequenceClassification(Data2VecAudioPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1419,12 +1335,7 @@ class Data2VecAudioForSequenceClassification(Data2VecAudioPreTrainedModel): ) -@add_start_docstrings( - """ - Data2VecAudio Model with a frame classification head on top for tasks like Speaker Diarization. - """, - DATA2VEC_AUDIO_START_DOCSTRING, -) +@auto_docstring class Data2VecAudioForAudioFrameClassification(Data2VecAudioPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1469,13 +1380,7 @@ class Data2VecAudioForAudioFrameClassification(Data2VecAudioPreTrainedModel): for param in self.data2vec_audio.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1486,6 +1391,11 @@ class Data2VecAudioForAudioFrameClassification(Data2VecAudioPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1585,11 +1495,10 @@ class TDNNLayer(nn.Module): return hidden_states -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Data2VecAudio Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - DATA2VEC_AUDIO_START_DOCSTRING, + """ ) class Data2VecAudioForXVector(Data2VecAudioPreTrainedModel): def __init__(self, config): @@ -1653,13 +1562,7 @@ class Data2VecAudioForXVector(Data2VecAudioPreTrainedModel): return input_lengths - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1670,6 +1573,11 @@ class Data2VecAudioForXVector(Data2VecAudioPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/data2vec/modeling_data2vec_text.py b/src/transformers/models/data2vec/modeling_data2vec_text.py index 5f84eca754..a1d747476d 100644 --- a/src/transformers/models/data2vec/modeling_data2vec_text.py +++ b/src/transformers/models/data2vec/modeling_data2vec_text.py @@ -36,13 +36,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_data2vec_text import Data2VecTextConfig @@ -51,10 +45,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 2 -# General docstring -_CHECKPOINT_FOR_DOC = "facebook/data2vec-text-base" -_CONFIG_FOR_DOC = "Data2VecTextConfig" - # Copied from transformers.models.roberta.modeling_roberta.RobertaEmbeddings with Roberta->Data2VecText class Data2VecTextForTextEmbeddings(nn.Module): @@ -581,12 +571,8 @@ class Data2VecTextPooler(nn.Module): return pooled_output +@auto_docstring class Data2VecTextPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Data2VecTextConfig base_model_prefix = "data2vec_text" supports_gradient_checkpointing = True @@ -611,79 +597,7 @@ class Data2VecTextPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -DATA2VECTEXT_START_DOCSTRING = r""" - Data2VecText was proposed in [data2vec: A General Framework for Self-supervised Learning in Speech, Vision and - Language](https://arxiv.org/pdf/2202.03555) by Alexei Baevski, Wei-Ning Hsu, Qiantong Xu, Arun Babu, Jiatao Gu and - Michael Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Data2VecTextConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DATA2VECTEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Data2VecText Model for text transformer outputting raw hidden-states without any specific head on top.", - DATA2VECTEXT_START_DOCSTRING, -) +@auto_docstring class Data2VecTextModel(Data2VecTextPreTrainedModel): """ @@ -701,6 +615,10 @@ class Data2VecTextModel(Data2VecTextPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -726,12 +644,7 @@ class Data2VecTextModel(Data2VecTextPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.clap.modeling_clap.ClapTextModel.forward def forward( self, @@ -749,26 +662,6 @@ class Data2VecTextModel(Data2VecTextPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -864,8 +757,10 @@ class Data2VecTextModel(Data2VecTextPreTrainedModel): ) -@add_start_docstrings( - """Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.""", DATA2VECTEXT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + Data2VecText Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -888,8 +783,7 @@ class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -909,31 +803,10 @@ class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1005,7 +878,7 @@ class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings("""data2vec Model with a `language modeling` head on top.""", DATA2VECTEXT_START_DOCSTRING) +@auto_docstring class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1030,13 +903,7 @@ class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1057,8 +924,6 @@ class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to *{}*): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1129,12 +994,11 @@ class Data2VecTextLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - DATA2VECTEXT_START_DOCSTRING, + """ ) class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel): def __init__(self, config): @@ -1148,12 +1012,7 @@ class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1226,13 +1085,7 @@ class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel): ) -@add_start_docstrings( - """ - Data2VecText Model with a multiple choice classification head on top (a linear layer on top of the pooled output - and a softmax) e.g. for RocStories/SWAG tasks. - """, - DATA2VECTEXT_START_DOCSTRING, -) +@auto_docstring class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1244,14 +1097,7 @@ class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1266,10 +1112,34 @@ class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1320,13 +1190,7 @@ class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel): ) -@add_start_docstrings( - """ - Data2VecText Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - DATA2VECTEXT_START_DOCSTRING, -) +@auto_docstring class Data2VecTextForTokenClassification(Data2VecTextPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1342,12 +1206,7 @@ class Data2VecTextForTokenClassification(Data2VecTextPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1426,13 +1285,7 @@ class Data2VecTextClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - Data2VecText Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - DATA2VECTEXT_START_DOCSTRING, -) +@auto_docstring class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1444,12 +1297,7 @@ class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DATA2VECTEXT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1464,16 +1312,6 @@ class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.data2vec_text( diff --git a/src/transformers/models/data2vec/modeling_data2vec_vision.py b/src/transformers/models/data2vec/modeling_data2vec_vision.py index 7ea0ffd39d..33dae045d4 100644 --- a/src/transformers/models/data2vec/modeling_data2vec_vision.py +++ b/src/transformers/models/data2vec/modeling_data2vec_vision.py @@ -34,30 +34,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import compile_compatible_method_lru_cache, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_data2vec_vision import Data2VecVisionConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "Data2VecVisionConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/data2vec-vision-base" -_EXPECTED_OUTPUT_SHAPE = [1, 197, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/data2vec-vision-base-ft1k" -_IMAGE_CLASS_EXPECTED_OUTPUT = "remote control, remote" - @dataclass # Copied from transformers.models.beit.modeling_beit.BeitModelOutputWithPooling with Beit->Data2VecVision @@ -754,13 +736,9 @@ class Data2VecVisionEncoder(nn.Module): ) +@auto_docstring # Copied from transformers.models.beit.modeling_beit.BeitPreTrainedModel with Beit->Data2VecVision,beit->data2vec_vision class Data2VecVisionPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Data2VecVisionConfig base_model_prefix = "data2vec_vision" main_input_name = "pixel_values" @@ -798,49 +776,14 @@ class Data2VecVisionPreTrainedModel(PreTrainedModel): module.lambda_2.data.fill_(self.config.layer_scale_init_value) -DATA2VEC_VISION_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Data2VecVisionConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DATA2VEC_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`BeitImageProcessor.__call__`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Data2VecVision Model transformer outputting raw hidden-states without any specific head on top.", - DATA2VEC_VISION_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.beit.modeling_beit.BeitModel with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,True->False class Data2VecVisionModel(Data2VecVisionPreTrainedModel): def __init__(self, config: Data2VecVisionConfig, add_pooling_layer: bool = False) -> None: + r""" + add_pooling_layer (bool, *optional*, defaults to `False`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -866,14 +809,7 @@ class Data2VecVisionModel(Data2VecVisionPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DATA2VEC_VISION_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Data2VecVisionModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -949,12 +885,11 @@ class Data2VecVisionPooler(nn.Module): return pooled_output -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Data2VecVision Model transformer with an image classification head on top (a linear layer on top of the average of the final hidden states of the patch tokens) e.g. for ImageNet. - """, - DATA2VEC_VISION_START_DOCSTRING, + """ ) # Copied from transformers.models.beit.modeling_beit.BeitForImageClassification with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,beit->data2vec_vision class Data2VecVisionForImageClassification(Data2VecVisionPreTrainedModel): @@ -970,13 +905,7 @@ class Data2VecVisionForImageClassification(Data2VecVisionPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DATA2VEC_VISION_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1287,12 +1216,7 @@ class Data2VecVisionFCNHead(nn.Module): return output -@add_start_docstrings( - """ - Data2VecVision Model transformer with a semantic segmentation head on top e.g. for ADE20k, CityScapes. - """, - DATA2VEC_VISION_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.beit.modeling_beit.BeitForSemanticSegmentation with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,microsoft/beit-base-finetuned-ade-640-640->facebook/data2vec-vision-base,beit->data2vec_vision class Data2VecVisionForSemanticSegmentation(Data2VecVisionPreTrainedModel): def __init__(self, config: Data2VecVisionConfig) -> None: @@ -1346,8 +1270,7 @@ class Data2VecVisionForSemanticSegmentation(Data2VecVisionPreTrainedModel): return loss - @add_start_docstrings_to_model_forward(DATA2VEC_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1363,8 +1286,6 @@ class Data2VecVisionForSemanticSegmentation(Data2VecVisionPreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/data2vec/modular_data2vec_audio.py b/src/transformers/models/data2vec/modular_data2vec_audio.py index 052f22a960..58934d2e86 100644 --- a/src/transformers/models/data2vec/modular_data2vec_audio.py +++ b/src/transformers/models/data2vec/modular_data2vec_audio.py @@ -4,19 +4,8 @@ import torch from torch import nn from ...activations import ACT2FN -from ...modeling_outputs import ( - CausalLMOutput, - SequenceClassifierOutput, - TokenClassifierOutput, - Wav2Vec2BaseModelOutput, - XVectorOutput, -) +from ...modeling_outputs import Wav2Vec2BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, -) from ..wav2vec2.modeling_wav2vec2 import ( Wav2Vec2Adapter, Wav2Vec2Encoder, @@ -33,20 +22,6 @@ from ..wav2vec2.modeling_wav2vec2 import ( from .configuration_data2vec_audio import Data2VecAudioConfig -_HIDDEN_STATES_START_POSITION = 2 - -# General docstring -_CONFIG_FOR_DOC = "Data2VecAudioConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/data2vec-audio-base-960h" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 66.95 - - class Data2VecAudioConvLayer(nn.Module): def __init__(self, config, layer_id=0): super().__init__() @@ -143,11 +118,6 @@ class Data2VecAudioAdapter(Wav2Vec2Adapter): class Data2VecAudioPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Data2VecAudioConfig base_model_prefix = "data2vec_audio" main_input_name = "input_values" @@ -190,67 +160,9 @@ class Data2VecAudioPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel): raise AttributeError("Not needed for Data2VecAudio") -DATA2VEC_AUDIO_START_DOCSTRING = r""" - Data2VecAudio was proposed in [data2vec: A General Framework for Self-supervised Learning in Speech, Vision and - Language](https://arxiv.org/pdf/2202.03555) by Alexei Baevski, Wei-Ning Hsu, Qiantong Xu, Arun Babu, Jiatao Gu and - Michael Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Data2VecAudioConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DATA2VEC_AUDIO_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should be passed if the corresponding processor has `config.return_attention_mask == - True`, which is the case for all pre-trained Data2Vec Audio models. Be aware that that even with - `attention_mask`, zero-padded inputs will have slightly different outputs compared to non-padded inputs - because there are more than one convolutional layer in the positional encodings. For a more detailed - explanation, see [here](https://github.com/huggingface/transformers/issues/25621#issuecomment-1713759349). - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - Data2VecAudioBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare Data2VecAudio Model transformer outputting raw hidden-states without any specific head on top.", - DATA2VEC_AUDIO_START_DOCSTRING, -) class Data2VecAudioModel(Data2VecAudioPreTrainedModel, Wav2Vec2Model): def __init__(self, config: Data2VecAudioConfig): Data2VecAudioPreTrainedModel.__init__(config) @@ -279,22 +191,10 @@ class Data2VecAudioModel(Data2VecAudioPreTrainedModel, Wav2Vec2Model): """ self.feature_extractor._freeze_parameters() - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Data2VecAudioBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) def forward(self, **super_kwargs): return super().forward(**super_kwargs) -@add_start_docstrings( - """Data2VecAudio Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - DATA2VEC_AUDIO_START_DOCSTRING, -) class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel, Wav2Vec2ForCTC): def __init__(self, config): Data2VecAudioPreTrainedModel.__init__(config) @@ -323,71 +223,20 @@ class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel, Wav2Vec2ForCTC): def tie_weights(self): raise AttributeError("Not needed for Data2VecAudio") - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) def forward(self, **super_kwargs): return super().forward(**super_kwargs) -@add_start_docstrings( - """ - Data2VecAudio Model with a sequence classification head on top (a linear layer over the pooled output) for tasks - like SUPERB Keyword Spotting. - """, - DATA2VEC_AUDIO_START_DOCSTRING, -) class Data2VecAudioForSequenceClassification(Wav2Vec2ForSequenceClassification): - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) + pass -@add_start_docstrings( - """ - Data2VecAudio Model with a frame classification head on top for tasks like Speaker Diarization. - """, - DATA2VEC_AUDIO_START_DOCSTRING, -) class Data2VecAudioForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification): - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) + pass -@add_start_docstrings( - """ - Data2VecAudio Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - DATA2VEC_AUDIO_START_DOCSTRING, -) class Data2VecAudioForXVector(Wav2Vec2ForXVector): - @add_start_docstrings_to_model_forward(DATA2VEC_AUDIO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) + pass __all__ = [ diff --git a/src/transformers/models/dbrx/modeling_dbrx.py b/src/transformers/models/dbrx/modeling_dbrx.py index ccde31d341..050476355c 100644 --- a/src/transformers/models/dbrx/modeling_dbrx.py +++ b/src/transformers/models/dbrx/modeling_dbrx.py @@ -28,13 +28,7 @@ from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, logging from .configuration_dbrx import DbrxConfig @@ -49,8 +43,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DbrxConfig" - class DbrxRotaryEmbedding(nn.Module): def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): @@ -807,27 +799,7 @@ class DbrxBlock(nn.Module): return outputs -DBRX_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DbrxConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare DBRX Model outputting raw hidden-states without any specific head on top.", - DBRX_START_DOCSTRING, -) +@auto_docstring class DbrxPreTrainedModel(PreTrainedModel): config_class = DbrxConfig base_model_prefix = "transformer" @@ -860,88 +832,7 @@ class DbrxPreTrainedModel(PreTrainedModel): module.w2.data.normal_(mean=0.0, std=std) -DBRX_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare DBRX Model outputting raw hidden-states without any specific head on top.", - DBRX_START_DOCSTRING, -) +@auto_docstring class DbrxModel(DbrxPreTrainedModel): """Transformer decoder consisting of *config.num_hidden_layers*. Each layer is a [`DbrxBlock`] layer. @@ -971,7 +862,7 @@ class DbrxModel(DbrxPreTrainedModel): def set_input_embeddings(self, value: nn.Embedding): self.wte = value - @add_start_docstrings_to_model_forward(DBRX_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1236,7 +1127,11 @@ class DbrxModel(DbrxPreTrainedModel): return causal_mask -@add_start_docstrings("The DBRX Model transformer for causal language modeling.", DBRX_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + The DBRX Model transformer for causal language modeling. + """ +) class DbrxForCausalLM(DbrxPreTrainedModel, GenerationMixin): def __init__(self, config: DbrxConfig): super().__init__(config) @@ -1268,8 +1163,7 @@ class DbrxForCausalLM(DbrxPreTrainedModel, GenerationMixin): def get_decoder(self) -> DbrxModel: return self.transformer - @add_start_docstrings_to_model_forward(DBRX_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1288,19 +1182,10 @@ class DbrxForCausalLM(DbrxPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, MoeCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/deberta/modeling_deberta.py b/src/transformers/models/deberta/modeling_deberta.py index c574460513..b23c189bef 100644 --- a/src/transformers/models/deberta/modeling_deberta.py +++ b/src/transformers/models/deberta/modeling_deberta.py @@ -30,25 +30,11 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_deberta import DebertaConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DebertaConfig" -_CHECKPOINT_FOR_DOC = "microsoft/deberta-base" - -# Masked LM docstring -_CHECKPOINT_FOR_MASKED_LM = "lsanochkin/deberta-large-feedback" -_MASKED_LM_EXPECTED_OUTPUT = "' Paris'" -_MASKED_LM_EXPECTED_LOSS = "0.54" - -# QuestionAnswering docstring -_CHECKPOINT_FOR_QA = "Palak/microsoft_deberta-large_squad" -_QA_EXPECTED_OUTPUT = "' a nice puppet'" -_QA_EXPECTED_LOSS = 0.14 -_QA_TARGET_START_INDEX = 12 -_QA_TARGET_END_INDEX = 14 class DebertaLayerNorm(nn.Module): @@ -632,12 +618,8 @@ class DebertaEncoder(nn.Module): ) +@auto_docstring class DebertaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DebertaConfig base_model_prefix = "deberta" _keys_to_ignore_on_load_unexpected = ["position_embeddings"] @@ -665,71 +647,7 @@ class DebertaPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -DEBERTA_START_DOCSTRING = r""" - The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled - Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build - on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two - improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data. - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - - Parameters: - config ([`DebertaConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DEBERTA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare DeBERTa Model transformer outputting raw hidden-states without any specific head on top.", - DEBERTA_START_DOCSTRING, -) +@auto_docstring class DebertaModel(DebertaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -754,12 +672,7 @@ class DebertaModel(DebertaPreTrainedModel): """ raise NotImplementedError("The prune function is not implemented in DeBERTa model.") - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -933,7 +846,7 @@ class DebertaOnlyMLMHead(nn.Module): return prediction_scores -@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top.""", DEBERTA_START_DOCSTRING) +@auto_docstring class DebertaForMaskedLM(DebertaPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -964,15 +877,7 @@ class DebertaForMaskedLM(DebertaPreTrainedModel): self.lm_predictions.lm_head.dense = new_embeddings self.lm_predictions.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_MASKED_LM, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="[MASK]", - expected_output=_MASKED_LM_EXPECTED_OUTPUT, - expected_loss=_MASKED_LM_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1050,12 +955,11 @@ class ContextPooler(nn.Module): return self.config.hidden_size -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DeBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - DEBERTA_START_DOCSTRING, + """ ) class DebertaForSequenceClassification(DebertaPreTrainedModel): def __init__(self, config): @@ -1082,12 +986,7 @@ class DebertaForSequenceClassification(DebertaPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.deberta.set_input_embeddings(new_embeddings) - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1168,13 +1067,7 @@ class DebertaForSequenceClassification(DebertaPreTrainedModel): ) -@add_start_docstrings( - """ - DeBERTa Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - DEBERTA_START_DOCSTRING, -) +@auto_docstring class DebertaForTokenClassification(DebertaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1187,12 +1080,7 @@ class DebertaForTokenClassification(DebertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1241,13 +1129,7 @@ class DebertaForTokenClassification(DebertaPreTrainedModel): ) -@add_start_docstrings( - """ - DeBERTa Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - DEBERTA_START_DOCSTRING, -) +@auto_docstring class DebertaForQuestionAnswering(DebertaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1259,16 +1141,7 @@ class DebertaForQuestionAnswering(DebertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_QA, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_QA_EXPECTED_OUTPUT, - expected_loss=_QA_EXPECTED_LOSS, - qa_target_start_index=_QA_TARGET_START_INDEX, - qa_target_end_index=_QA_TARGET_END_INDEX, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1282,16 +1155,6 @@ class DebertaForQuestionAnswering(DebertaPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.deberta( diff --git a/src/transformers/models/deberta_v2/modeling_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_deberta_v2.py index 576c4ce879..06462ed477 100644 --- a/src/transformers/models/deberta_v2/modeling_deberta_v2.py +++ b/src/transformers/models/deberta_v2/modeling_deberta_v2.py @@ -32,17 +32,12 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_deberta_v2 import DebertaV2Config logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DebertaV2Config" -_CHECKPOINT_FOR_DOC = "microsoft/deberta-v2-xlarge" -_QA_TARGET_START_INDEX = 2 -_QA_TARGET_END_INDEX = 9 - # Copied from transformers.models.deberta.modeling_deberta.DebertaSelfOutput with DebertaLayerNorm->LayerNorm class DebertaV2SelfOutput(nn.Module): @@ -703,12 +698,8 @@ class DebertaV2Encoder(nn.Module): ) +@auto_docstring class DebertaV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DebertaV2Config base_model_prefix = "deberta" _keys_to_ignore_on_load_unexpected = ["position_embeddings"] @@ -733,71 +724,7 @@ class DebertaV2PreTrainedModel(PreTrainedModel): module.bias.data.zero_() -DEBERTA_START_DOCSTRING = r""" - The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled - Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build - on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two - improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data. - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - - Parameters: - config ([`DebertaV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DEBERTA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare DeBERTa Model transformer outputting raw hidden-states without any specific head on top.", - DEBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.deberta.modeling_deberta.DebertaModel with Deberta->DebertaV2 class DebertaV2Model(DebertaV2PreTrainedModel): def __init__(self, config): @@ -823,12 +750,7 @@ class DebertaV2Model(DebertaV2PreTrainedModel): """ raise NotImplementedError("The prune function is not implemented in DeBERTa model.") - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1000,7 +922,7 @@ class DebertaV2OnlyMLMHead(nn.Module): return prediction_scores -@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top.""", DEBERTA_START_DOCSTRING) +@auto_docstring class DebertaV2ForMaskedLM(DebertaV2PreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] _keys_to_ignore_on_load_unexpected = r"mask_predictions.*" @@ -1031,13 +953,7 @@ class DebertaV2ForMaskedLM(DebertaV2PreTrainedModel): self.lm_predictions.lm_head.dense = new_embeddings self.lm_predictions.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="[MASK]", - ) + @auto_docstring # Copied from transformers.models.deberta.modeling_deberta.DebertaForMaskedLM.forward with Deberta->DebertaV2 def forward( self, @@ -1117,12 +1033,11 @@ class ContextPooler(nn.Module): return self.config.hidden_size -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DeBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - DEBERTA_START_DOCSTRING, + """ ) class DebertaV2ForSequenceClassification(DebertaV2PreTrainedModel): def __init__(self, config): @@ -1149,12 +1064,7 @@ class DebertaV2ForSequenceClassification(DebertaV2PreTrainedModel): def set_input_embeddings(self, new_embeddings): self.deberta.set_input_embeddings(new_embeddings) - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.deberta.modeling_deberta.DebertaForSequenceClassification.forward with Deberta->DebertaV2 def forward( self, @@ -1236,13 +1146,7 @@ class DebertaV2ForSequenceClassification(DebertaV2PreTrainedModel): ) -@add_start_docstrings( - """ - DeBERTa Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - DEBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.deberta.modeling_deberta.DebertaForTokenClassification with Deberta->DebertaV2 class DebertaV2ForTokenClassification(DebertaV2PreTrainedModel): def __init__(self, config): @@ -1256,12 +1160,7 @@ class DebertaV2ForTokenClassification(DebertaV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1310,13 +1209,7 @@ class DebertaV2ForTokenClassification(DebertaV2PreTrainedModel): ) -@add_start_docstrings( - """ - DeBERTa Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - DEBERTA_START_DOCSTRING, -) +@auto_docstring class DebertaV2ForQuestionAnswering(DebertaV2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1328,14 +1221,7 @@ class DebertaV2ForQuestionAnswering(DebertaV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - qa_target_start_index=_QA_TARGET_START_INDEX, - qa_target_end_index=_QA_TARGET_END_INDEX, - ) + @auto_docstring # Copied from transformers.models.deberta.modeling_deberta.DebertaForQuestionAnswering.forward with Deberta->DebertaV2 def forward( self, @@ -1350,16 +1236,6 @@ class DebertaV2ForQuestionAnswering(DebertaV2PreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.deberta( @@ -1410,13 +1286,7 @@ class DebertaV2ForQuestionAnswering(DebertaV2PreTrainedModel): ) -@add_start_docstrings( - """ - DeBERTa Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - DEBERTA_START_DOCSTRING, -) +@auto_docstring class DebertaV2ForMultipleChoice(DebertaV2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1441,12 +1311,7 @@ class DebertaV2ForMultipleChoice(DebertaV2PreTrainedModel): def set_input_embeddings(self, new_embeddings): self.deberta.set_input_embeddings(new_embeddings) - @add_start_docstrings_to_model_forward(DEBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/decision_transformer/modeling_decision_transformer.py b/src/transformers/models/decision_transformer/modeling_decision_transformer.py index ab2a302405..c577d6f17c 100755 --- a/src/transformers/models/decision_transformer/modeling_decision_transformer.py +++ b/src/transformers/models/decision_transformer/modeling_decision_transformer.py @@ -30,10 +30,8 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import Conv1D, find_pruneable_heads_and_indices, prune_conv1d_layer from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from ...utils.deprecation import deprecate_kwarg from .configuration_decision_transformer import DecisionTransformerConfig @@ -41,9 +39,6 @@ from .configuration_decision_transformer import DecisionTransformerConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "edbeeching/decision-transformer-gym-hopper-medium" -_CONFIG_FOR_DOC = "DecisionTransformerConfig" - # Copied from transformers.models.gpt2.modeling_gpt2.load_tf_weights_in_gpt2 def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path): @@ -449,12 +444,8 @@ class DecisionTransformerGPT2Block(nn.Module): return outputs +@auto_docstring class DecisionTransformerGPT2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DecisionTransformerConfig load_tf_weights = load_tf_weights_in_gpt2 base_model_prefix = "transformer" @@ -795,36 +786,11 @@ class DecisionTransformerPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -DECISION_TRANSFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~DecisionTransformerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DECISION_TRANSFORMER_INPUTS_DOCSTRING = r""" - Args: - states (`torch.FloatTensor` of shape `(batch_size, episode_length, state_dim)`): - The states for each step in the trajectory - actions (`torch.FloatTensor` of shape `(batch_size, episode_length, act_dim)`): - The actions taken by the "expert" policy for the current state, these are masked for auto regressive - prediction - rewards (`torch.FloatTensor` of shape `(batch_size, episode_length, 1)`): - The rewards for each state, action - returns_to_go (`torch.FloatTensor` of shape `(batch_size, episode_length, 1)`): - The returns for each state in the trajectory - timesteps (`torch.LongTensor` of shape `(batch_size, episode_length)`): - The timestep for each step in the trajectory - attention_mask (`torch.FloatTensor` of shape `(batch_size, episode_length)`): - Masking, used to mask the actions when performing autoregressive prediction -""" - - -@add_start_docstrings("The Decision Transformer Model", DECISION_TRANSFORMER_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + The Decision Transformer Model + """ +) class DecisionTransformerModel(DecisionTransformerPreTrainedModel): """ @@ -858,8 +824,7 @@ class DecisionTransformerModel(DecisionTransformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DECISION_TRANSFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=DecisionTransformerOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, states: Optional[torch.FloatTensor] = None, @@ -873,7 +838,17 @@ class DecisionTransformerModel(DecisionTransformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DecisionTransformerOutput]: r""" - Returns: + states (`torch.FloatTensor` of shape `(batch_size, episode_length, state_dim)`): + The states for each step in the trajectory + actions (`torch.FloatTensor` of shape `(batch_size, episode_length, act_dim)`): + The actions taken by the "expert" policy for the current state, these are masked for auto regressive + prediction + rewards (`torch.FloatTensor` of shape `(batch_size, episode_length, 1)`): + The rewards for each state, action + returns_to_go (`torch.FloatTensor` of shape `(batch_size, episode_length, 1)`): + The returns for each state in the trajectory + timesteps (`torch.LongTensor` of shape `(batch_size, episode_length)`): + The timestep for each step in the trajectory Examples: diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index b54ff78302..8388f743f2 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -22,15 +22,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_deepseek_v3 import DeepseekV3Config @@ -41,7 +33,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DeepseekV3Config" @use_kernel_forward_from_hub("RMSNorm") @@ -511,27 +502,7 @@ class DeepseekV3DecoderLayer(GradientCheckpointingLayer): return outputs -DEEPSEEK_V3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DeepseekV3Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare DeepseekV3 Model outputting raw hidden-states without any specific head on top.", - DEEPSEEK_V3_START_DOCSTRING, -) +@auto_docstring class DeepseekV3PreTrainedModel(PreTrainedModel): config_class = DeepseekV3Config base_model_prefix = "model" @@ -562,88 +533,8 @@ class DeepseekV3PreTrainedModel(PreTrainedModel): module.weight.data.normal_(mean=0.0, std=std) -DEEPSEEK_V3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare DeepseekV3 Model outputting raw hidden-states without any specific head on top.", - DEEPSEEK_V3_START_DOCSTRING, -) +@auto_docstring class DeepseekV3Model(DeepseekV3PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DeepseekV3DecoderLayer`] - - Args: - config: DeepseekV3Config - """ - _keys_to_ignore_on_load_unexpected = [r"model\.layers\.61.*"] def __init__(self, config: DeepseekV3Config): @@ -669,7 +560,7 @@ class DeepseekV3Model(DeepseekV3PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(DEEPSEEK_V3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -891,6 +782,7 @@ class DeepseekV3Model(DeepseekV3PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -924,8 +816,7 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(DEEPSEEK_V3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -942,19 +833,10 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr_fast.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr_fast.py index 2a820e07f7..cd25fb7a9d 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr_fast.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr_fast.py @@ -9,8 +9,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union from ...image_processing_utils import BatchFeature, get_size_dict from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, @@ -33,7 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -59,6 +57,26 @@ logger = logging.get_logger(__name__) class DeformableDetrFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): + Data format of the annotations. One of "coco_detection" or "coco_panoptic". + do_convert_annotations (`bool`, *optional*, defaults to `True`): + Controls whether to convert the annotations to the format expected by the DEFORMABLE_DETR model. Converts the + bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. + Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with zeros. + If `pad_size` is provided, the image will be padded to the specified dimensions. + Otherwise, the image will be padded to the maximum height and width of the batch. + pad_size (`Dict[str, int]`, *optional*): + The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size + provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest + height and width in the batch. + return_segmentation_masks (`bool`, *optional*, defaults to `False`): + Whether to return segmentation masks. + """ + format: Optional[Union[str, AnnotationFormat]] do_convert_annotations: Optional[bool] do_pad: Optional[bool] @@ -266,29 +284,7 @@ def prepare_coco_panoptic_annotation( return new_target -@add_start_docstrings( - "Constructs a fast DeformableDetr image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DEFORMABLE_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - """, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class DeformableDetrImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR @@ -562,9 +558,15 @@ class DeformableDetrImageProcessorFast(BaseImageProcessorFast): return image, pixel_mask, annotation - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ + @auto_docstring + def preprocess( + self, + images: ImageInput, + annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, + masks_path: Optional[Union[str, pathlib.Path]] = None, + **kwargs: Unpack[DeformableDetrFastImageProcessorKwargs], + ) -> BatchFeature: + r""" annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: @@ -578,32 +580,9 @@ class DeformableDetrImageProcessorFast(BaseImageProcessorFast): - "file_name" (`str`): The file name of the image. format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DEFORMABLE_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. masks_path (`str` or `pathlib.Path`, *optional*): Path to the directory containing the segmentation masks. - """, - ) - def preprocess( - self, - images: ImageInput, - annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, - masks_path: Optional[Union[str, pathlib.Path]] = None, - **kwargs: Unpack[DeformableDetrFastImageProcessorKwargs], - ) -> BatchFeature: + """ if "pad_and_return_pixel_mask" in kwargs: kwargs["do_pad"] = kwargs.pop("pad_and_return_pixel_mask") logger.warning_once( diff --git a/src/transformers/models/deformable_detr/modeling_deformable_detr.py b/src/transformers/models/deformable_detr/modeling_deformable_detr.py index a540bdc5d7..1dbcb2ffb3 100755 --- a/src/transformers/models/deformable_detr/modeling_deformable_detr.py +++ b/src/transformers/models/deformable_detr/modeling_deformable_detr.py @@ -32,11 +32,9 @@ from ...modeling_utils import PreTrainedModel from ...pytorch_utils import meshgrid from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_timm_available, logging, - replace_return_docstrings, requires_backends, ) from ...utils.backbone_utils import load_backbone @@ -52,9 +50,6 @@ if is_timm_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DeformableDetrConfig" -_CHECKPOINT_FOR_DOC = "sensetime/deformable-detr" - @use_kernel_forward_from_hub("MultiScaleDeformableAttention") class MultiScaleDeformableAttention(nn.Module): @@ -966,6 +961,7 @@ class DeformableDetrDecoderLayer(nn.Module): return outputs +@auto_docstring class DeformableDetrPreTrainedModel(PreTrainedModel): config_class = DeformableDetrConfig base_model_prefix = "model" @@ -1022,61 +1018,6 @@ class DeformableDetrPreTrainedModel(PreTrainedModel): nn.init.normal_(module.level_embed) -DEFORMABLE_DETR_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DeformableDetrConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DEFORMABLE_DETR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`AutoImageProcessor`]. See [`DeformableDetrImageProcessor.__call__`] - for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): - Not used by default. Can be used to mask object queries. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - class DeformableDetrEncoder(DeformableDetrPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* deformable attention layers. Each layer is a @@ -1417,12 +1358,11 @@ class DeformableDetrDecoder(DeformableDetrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The bare Deformable DETR Model (consisting of a backbone and encoder-decoder Transformer) outputting raw hidden-states without any specific head on top. - """, - DEFORMABLE_DETR_START_DOCSTRING, + """ ) class DeformableDetrModel(DeformableDetrPreTrainedModel): def __init__(self, config: DeformableDetrConfig): @@ -1595,8 +1535,7 @@ class DeformableDetrModel(DeformableDetrPreTrainedModel): object_query = self.enc_output_norm(self.enc_output(object_query)) return object_query, output_proposals - @add_start_docstrings_to_model_forward(DEFORMABLE_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DeformableDetrModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1610,7 +1549,14 @@ class DeformableDetrModel(DeformableDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DeformableDetrModelOutput]: r""" - Returns: + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. Examples: @@ -1823,12 +1769,11 @@ class DeformableDetrMLPPredictionHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Deformable DETR Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on top, for tasks such as COCO detection. - """, - DEFORMABLE_DETR_START_DOCSTRING, + """ ) class DeformableDetrForObjectDetection(DeformableDetrPreTrainedModel): # When using clones, all layers > 0 will be clones, but layer 0 *is* required @@ -1868,8 +1813,7 @@ class DeformableDetrForObjectDetection(DeformableDetrPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEFORMABLE_DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DeformableDetrObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1884,14 +1828,20 @@ class DeformableDetrForObjectDetection(DeformableDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DeformableDetrObjectDetectionOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/deit/image_processing_deit_fast.py b/src/transformers/models/deit/image_processing_deit_fast.py index 28cd6539df..3aafeaf50c 100644 --- a/src/transformers/models/deit/image_processing_deit_fast.py +++ b/src/transformers/models/deit/image_processing_deit_fast.py @@ -14,19 +14,16 @@ # limitations under the License. """Fast Image processor class for DeiT.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import ( IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling, ) -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast DeiT image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class DeiTImageProcessorFast(BaseImageProcessorFast): # To be checked against the slow image processor # None values left after checking can be removed diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py index cb88bca353..1b2ba10365 100644 --- a/src/transformers/models/deit/modeling_deit.py +++ b/src/transformers/models/deit/modeling_deit.py @@ -32,31 +32,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_deit import DeiTConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "DeiTConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/deit-base-distilled-patch16-224" -_EXPECTED_OUTPUT_SHAPE = [1, 198, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/deit-base-distilled-patch16-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - class DeiTEmbeddings(nn.Module): """ @@ -460,12 +441,8 @@ class DeiTEncoder(nn.Module): ) +@auto_docstring class DeiTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DeiTConfig base_model_prefix = "deit" main_input_name = "pixel_values" @@ -495,48 +472,15 @@ class DeiTPreTrainedModel(PreTrainedModel): module.mask_token.data.zero_() -DEIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`DeiTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DEIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`DeiTImageProcessor.__call__`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - - -@add_start_docstrings( - "The bare DeiT Model transformer outputting raw hidden-states without any specific head on top.", - DEIT_START_DOCSTRING, -) +@auto_docstring class DeiTModel(DeiTPreTrainedModel): def __init__(self, config: DeiTConfig, add_pooling_layer: bool = True, use_mask_token: bool = False) -> None: + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config @@ -560,14 +504,7 @@ class DeiTModel(DeiTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -646,8 +583,9 @@ class DeiTPooler(nn.Module): return pooled_output -@add_start_docstrings( - """DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886). +@auto_docstring( + custom_intro=""" + DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886). @@ -655,8 +593,7 @@ class DeiTPooler(nn.Module): directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - """, - DEIT_START_DOCSTRING, + """ ) class DeiTForMaskedImageModeling(DeiTPreTrainedModel): def __init__(self, config: DeiTConfig) -> None: @@ -676,8 +613,7 @@ class DeiTForMaskedImageModeling(DeiTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -692,8 +628,6 @@ class DeiTForMaskedImageModeling(DeiTPreTrainedModel): bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling @@ -765,12 +699,11 @@ class DeiTForMaskedImageModeling(DeiTPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - DEIT_START_DOCSTRING, + """ ) class DeiTForImageClassification(DeiTPreTrainedModel): def __init__(self, config: DeiTConfig) -> None: @@ -785,8 +718,7 @@ class DeiTForImageClassification(DeiTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -803,8 +735,6 @@ class DeiTForImageClassification(DeiTPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -912,8 +842,8 @@ class DeiTForImageClassificationWithTeacherOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet. @@ -921,8 +851,7 @@ class DeiTForImageClassificationWithTeacherOutput(ModelOutput): This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet supported. - """, - DEIT_START_DOCSTRING, + """ ) class DeiTForImageClassificationWithTeacher(DeiTPreTrainedModel): def __init__(self, config: DeiTConfig) -> None: @@ -942,13 +871,7 @@ class DeiTForImageClassificationWithTeacher(DeiTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=DeiTForImageClassificationWithTeacherOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/depth_anything/modeling_depth_anything.py b/src/transformers/models/depth_anything/modeling_depth_anything.py index 17b79134ad..71ff5df6ba 100644 --- a/src/transformers/models/depth_anything/modeling_depth_anything.py +++ b/src/transformers/models/depth_anything/modeling_depth_anything.py @@ -20,14 +20,9 @@ import torch import torch.utils.checkpoint from torch import nn -from ...file_utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) from ...modeling_outputs import DepthEstimatorOutput from ...modeling_utils import PreTrainedModel -from ...utils import logging +from ...utils import auto_docstring, logging from ...utils.backbone_utils import load_backbone from .configuration_depth_anything import DepthAnythingConfig @@ -35,34 +30,6 @@ from .configuration_depth_anything import DepthAnythingConfig logger = logging.get_logger(__name__) # General docstring -_CONFIG_FOR_DOC = "DepthAnythingConfig" - - -DEPTH_ANYTHING_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`DepthAnythingConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DEPTH_ANYTHING_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] - for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" class DepthAnythingReassembleLayer(nn.Module): @@ -242,12 +209,8 @@ class DepthAnythingFeatureFusionStage(nn.Module): # Modified from transformers.models.dpt.modeling_dpt.DPTPreTrainedModel with DPT->DepthAnything,dpt->depth_anything # avoiding sdpa and flash_attn_2 support, it's done in the backend +@auto_docstring class DepthAnythingPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DepthAnythingConfig base_model_prefix = "depth_anything" main_input_name = "pixel_values" @@ -360,11 +323,10 @@ class DepthAnythingDepthEstimationHead(nn.Module): return predicted_depth -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Depth Anything Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. - """, - DEPTH_ANYTHING_START_DOCSTRING, + """ ) class DepthAnythingForDepthEstimation(DepthAnythingPreTrainedModel): _no_split_modules = ["DPTViTEmbeddings"] @@ -379,8 +341,7 @@ class DepthAnythingForDepthEstimation(DepthAnythingPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEPTH_ANYTHING_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -393,8 +354,6 @@ class DepthAnythingForDepthEstimation(DepthAnythingPreTrainedModel): labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth depth estimation maps for computing the loss. - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, AutoModelForDepthEstimation diff --git a/src/transformers/models/depth_pro/image_processing_depth_pro.py b/src/transformers/models/depth_pro/image_processing_depth_pro.py index 978a32da7e..7d3440bde3 100644 --- a/src/transformers/models/depth_pro/image_processing_depth_pro.py +++ b/src/transformers/models/depth_pro/image_processing_depth_pro.py @@ -36,21 +36,18 @@ from ...image_utils import ( is_scaled_image, is_torch_available, make_list_of_images, - pil_torch_interpolation_mapping, to_numpy_array, valid_images, ) -from ...utils import ( - TensorType, - filter_out_non_signature_kwargs, - logging, - requires_backends, -) +from ...utils import TensorType, filter_out_non_signature_kwargs, is_torchvision_available, logging, requires_backends if is_torch_available(): import torch +if is_torchvision_available(): + from ...image_utils import pil_torch_interpolation_mapping + logger = logging.get_logger(__name__) diff --git a/src/transformers/models/depth_pro/image_processing_depth_pro_fast.py b/src/transformers/models/depth_pro/image_processing_depth_pro_fast.py index a0a4cd96b8..669fcc00eb 100644 --- a/src/transformers/models/depth_pro/image_processing_depth_pro_fast.py +++ b/src/transformers/models/depth_pro/image_processing_depth_pro_fast.py @@ -18,7 +18,6 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from ...image_processing_base import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, group_images_by_shape, reorder_images, @@ -31,7 +30,7 @@ from ...image_utils import ( ) from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -60,10 +59,7 @@ if is_torchvision_available(): from torchvision.transforms import functional as F -@add_start_docstrings( - "Constructs a fast DepthPro image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class DepthProImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR diff --git a/src/transformers/models/depth_pro/modeling_depth_pro.py b/src/transformers/models/depth_pro/modeling_depth_pro.py index dbbc43502d..9eef406a66 100644 --- a/src/transformers/models/depth_pro/modeling_depth_pro.py +++ b/src/transformers/models/depth_pro/modeling_depth_pro.py @@ -22,16 +22,8 @@ import torch import torch.nn.functional as F from torch import nn -from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from ..auto import AutoModel from .configuration_depth_pro import DepthProConfig @@ -631,62 +623,10 @@ class DepthProNeck(nn.Module): # General docstring -_CONFIG_FOR_DOC = "DepthProConfig" - - -DEPTH_PRO_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`DepthProConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DEPTH_PRO_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - -DEPTH_PRO_FOR_DEPTH_ESTIMATION_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`DepthProConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. - use_fov_model (`bool`, *optional*, defaults to `True`): - Whether to use `DepthProFovModel` to generate the field of view. -""" +@auto_docstring class DepthProPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DepthProConfig base_model_prefix = "depth_pro" main_input_name = "pixel_values" @@ -712,10 +652,7 @@ class DepthProPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -@add_start_docstrings( - "The bare DepthPro Model transformer outputting raw hidden-states without any specific head on top.", - DEPTH_PRO_START_DOCSTRING, -) +@auto_docstring class DepthProModel(DepthProPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -728,8 +665,7 @@ class DepthProModel(DepthProPreTrainedModel): def get_input_embeddings(self): return self.encoder.image_encoder.model.get_input_embeddings() - @add_start_docstrings_to_model_forward(DEPTH_PRO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -739,8 +675,6 @@ class DepthProModel(DepthProPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, DepthProOutput]: r""" - Returns: - Examples: ```python @@ -1087,14 +1021,17 @@ class DepthProDepthEstimationHead(nn.Module): return predicted_depth -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DepthPro Model with a depth estimation head on top (consisting of 3 convolutional layers). - """, - DEPTH_PRO_FOR_DEPTH_ESTIMATION_START_DOCSTRING, + """ ) class DepthProForDepthEstimation(DepthProPreTrainedModel): def __init__(self, config, use_fov_model=None): + r""" + use_fov_model (bool, *optional*): + Whether to use the field of view model. + """ super().__init__(config) self.config = config self.use_fov_model = use_fov_model if use_fov_model is not None else self.config.use_fov_model @@ -1114,8 +1051,7 @@ class DepthProForDepthEstimation(DepthProPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DEPTH_PRO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DepthProDepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1129,8 +1065,6 @@ class DepthProForDepthEstimation(DepthProPreTrainedModel): labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth depth estimation maps for computing the loss. - Returns: - Examples: ```python diff --git a/src/transformers/models/detr/image_processing_detr_fast.py b/src/transformers/models/detr/image_processing_detr_fast.py index 419d099e91..cf9d4e52d8 100644 --- a/src/transformers/models/detr/image_processing_detr_fast.py +++ b/src/transformers/models/detr/image_processing_detr_fast.py @@ -21,8 +21,6 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Union from ...image_processing_utils import BatchFeature, get_size_dict from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, @@ -49,7 +47,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -284,6 +282,26 @@ def prepare_coco_panoptic_annotation( class DetrFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): + Data format of the annotations. One of "coco_detection" or "coco_panoptic". + do_convert_annotations (`bool`, *optional*, defaults to `True`): + Controls whether to convert the annotations to the format expected by the DETR model. Converts the + bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. + Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with zeros. + If `pad_size` is provided, the image will be padded to the specified dimensions. + Otherwise, the image will be padded to the maximum height and width of the batch. + pad_size (`Dict[str, int]`, *optional*): + The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size + provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest + height and width in the batch. + return_segmentation_masks (`bool`, *optional*, defaults to `False`): + Whether to return segmentation masks. + """ + format: Optional[Union[str, AnnotationFormat]] do_convert_annotations: Optional[bool] do_pad: Optional[bool] @@ -291,29 +309,7 @@ class DetrFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): return_segmentation_masks: Optional[bool] -@add_start_docstrings( - "Constructs a fast Detr image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - """, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class DetrImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR @@ -587,9 +583,15 @@ class DetrImageProcessorFast(BaseImageProcessorFast): return image, pixel_mask, annotation - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ + @auto_docstring + def preprocess( + self, + images: ImageInput, + annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, + masks_path: Optional[Union[str, pathlib.Path]] = None, + **kwargs: Unpack[DetrFastImageProcessorKwargs], + ) -> BatchFeature: + r""" annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: @@ -603,32 +605,9 @@ class DetrImageProcessorFast(BaseImageProcessorFast): - "file_name" (`str`): The file name of the image. format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. masks_path (`str` or `pathlib.Path`, *optional*): Path to the directory containing the segmentation masks. - """, - ) - def preprocess( - self, - images: ImageInput, - annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, - masks_path: Optional[Union[str, pathlib.Path]] = None, - **kwargs: Unpack[DetrFastImageProcessorKwargs], - ) -> BatchFeature: + """ if "pad_and_return_pixel_mask" in kwargs: kwargs["do_pad"] = kwargs.pop("pad_and_return_pixel_mask") logger.warning_once( diff --git a/src/transformers/models/detr/modeling_detr.py b/src/transformers/models/detr/modeling_detr.py index 989d68e75d..c23258fad1 100644 --- a/src/transformers/models/detr/modeling_detr.py +++ b/src/transformers/models/detr/modeling_detr.py @@ -27,11 +27,9 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttenti from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_timm_available, logging, - replace_return_docstrings, requires_backends, ) from ...utils.backbone_utils import load_backbone @@ -44,9 +42,6 @@ if is_timm_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DetrConfig" -_CHECKPOINT_FOR_DOC = "facebook/detr-resnet-50" - @dataclass class DetrDecoderOutput(BaseModelOutputWithCrossAttentions): @@ -787,6 +782,7 @@ class DetrDecoderLayer(nn.Module): return outputs +@auto_docstring class DetrPreTrainedModel(PreTrainedModel): config_class = DetrConfig base_model_prefix = "model" @@ -817,60 +813,6 @@ class DetrPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -DETR_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DetrConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DETR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`AutoImageProcessor`]. See [`DetrImageProcessor.__call__`] for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): - Not used by default. Can be used to mask object queries. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class DetrEncoder(DetrPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1161,12 +1103,11 @@ class DetrDecoder(DetrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The bare DETR Model (consisting of a backbone and encoder-decoder Transformer) outputting raw hidden-states without any specific head on top. - """, - DETR_START_DOCSTRING, + """ ) class DetrModel(DetrPreTrainedModel): def __init__(self, config: DetrConfig): @@ -1202,8 +1143,7 @@ class DetrModel(DetrPreTrainedModel): for name, param in self.backbone.conv_encoder.model.named_parameters(): param.requires_grad_(True) - @add_start_docstrings_to_model_forward(DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DetrModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1217,7 +1157,14 @@ class DetrModel(DetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DetrModelOutput]: r""" - Returns: + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. Examples: @@ -1351,12 +1298,11 @@ class DetrMLPPredictionHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DETR Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on top, for tasks such as COCO detection. - """, - DETR_START_DOCSTRING, + """ ) class DetrForObjectDetection(DetrPreTrainedModel): def __init__(self, config: DetrConfig): @@ -1376,8 +1322,7 @@ class DetrForObjectDetection(DetrPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DetrObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1392,14 +1337,20 @@ class DetrForObjectDetection(DetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DetrObjectDetectionOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python @@ -1490,13 +1441,11 @@ class DetrForObjectDetection(DetrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DETR Model (consisting of a backbone and encoder-decoder Transformer) with a segmentation head on top, for tasks such as COCO panoptic. - - """, - DETR_START_DOCSTRING, + """ ) class DetrForSegmentation(DetrPreTrainedModel): def __init__(self, config: DetrConfig): @@ -1519,8 +1468,7 @@ class DetrForSegmentation(DetrPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DetrSegmentationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1535,6 +1483,14 @@ class DetrForSegmentation(DetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], DetrSegmentationOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss, DICE/F-1 loss and Focal loss. List of dicts, each dictionary containing at least the following 3 keys: 'class_labels', 'boxes' and 'masks' (the class labels, @@ -1543,8 +1499,6 @@ class DetrForSegmentation(DetrPreTrainedModel): `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)` and the masks a `torch.FloatTensor` of shape `(number of bounding boxes in the image, height, width)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index f1393c694d..bc896482d3 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -48,16 +48,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_diffllama import DiffLlamaConfig @@ -69,9 +60,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "kajuma/DiffLlama-0.3B-handcut" -_CONFIG_FOR_DOC = "DiffLlamaConfig" - class DiffLlamaMLP(nn.Module): def __init__(self, config): @@ -579,27 +567,7 @@ class DiffLlamaDecoderLayer(GradientCheckpointingLayer): return outputs -DIFFLLAMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DiffLlamaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare DiffLlama Model outputting raw hidden-states without any specific head on top.", - DIFFLLAMA_START_DOCSTRING, -) +@auto_docstring class DiffLlamaPreTrainedModel(PreTrainedModel): config_class = DiffLlamaConfig base_model_prefix = "model" @@ -667,88 +635,8 @@ class DiffLlamaRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -DIFFLLAMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare DiffLlama Model outputting raw hidden-states without any specific head on top.", - DIFFLLAMA_START_DOCSTRING, -) +@auto_docstring class DiffLlamaModel(DiffLlamaPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DiffLlamaDecoderLayer`] - - Args: - config: DiffLlamaConfig - """ - def __init__(self, config: DiffLlamaConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -772,7 +660,7 @@ class DiffLlamaModel(DiffLlamaPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(DIFFLLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -994,6 +882,7 @@ class DiffLlamaModel(DiffLlamaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class DiffLlamaForCausalLM(DiffLlamaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -1027,8 +916,7 @@ class DiffLlamaForCausalLM(DiffLlamaPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(DIFFLLAMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1045,19 +933,10 @@ class DiffLlamaForCausalLM(DiffLlamaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1112,8 +991,8 @@ class DiffLlamaForCausalLM(DiffLlamaPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The DiffLlama Model transformer with a sequence classification head on top (linear layer). [`DiffLlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1124,8 +1003,7 @@ class DiffLlamaForCausalLM(DiffLlamaPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - DIFFLLAMA_START_DOCSTRING, + """ ) class DiffLlamaForSequenceClassification(DiffLlamaPreTrainedModel): def __init__(self, config): @@ -1144,7 +1022,7 @@ class DiffLlamaForSequenceClassification(DiffLlamaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(DIFFLLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1213,13 +1091,7 @@ class DiffLlamaForSequenceClassification(DiffLlamaPreTrainedModel): ) -@add_start_docstrings( - """ -The DiffLlama Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - DIFFLLAMA_START_DOCSTRING, -) +@auto_docstring class DiffLlamaForQuestionAnswering(DiffLlamaPreTrainedModel): base_model_prefix = "transformer" @@ -1238,7 +1110,7 @@ class DiffLlamaForQuestionAnswering(DiffLlamaPreTrainedModel): self.transformer.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(DIFFLLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1252,17 +1124,6 @@ class DiffLlamaForQuestionAnswering(DiffLlamaPreTrainedModel): output_hidden_states: Optional[bool] = None, **kwargs, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.transformer( input_ids, attention_mask=attention_mask, @@ -1293,13 +1154,7 @@ class DiffLlamaForQuestionAnswering(DiffLlamaPreTrainedModel): ) -@add_start_docstrings( - """ - The DiffLlama Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - DIFFLLAMA_START_DOCSTRING, -) +@auto_docstring class DiffLlamaForTokenClassification(DiffLlamaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1324,12 +1179,7 @@ class DiffLlamaForTokenClassification(DiffLlamaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(DIFFLLAMA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/dinat/modeling_dinat.py b/src/transformers/models/dinat/modeling_dinat.py index 8837372a84..79e46bd112 100644 --- a/src/transformers/models/dinat/modeling_dinat.py +++ b/src/transformers/models/dinat/modeling_dinat.py @@ -30,12 +30,9 @@ from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_laye from ...utils import ( ModelOutput, OptionalDependencyNotAvailable, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_natten_available, logging, - replace_return_docstrings, requires_backends, ) from ...utils.backbone_utils import BackboneMixin @@ -55,17 +52,6 @@ else: logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "DinatConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "shi-labs/dinat-mini-in1k-224" -_EXPECTED_OUTPUT_SHAPE = [1, 7, 7, 512] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "shi-labs/dinat-mini-in1k-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - # drop_path and DinatDropPath are from the timm library. @@ -618,12 +604,8 @@ class DinatEncoder(nn.Module): ) +@auto_docstring class DinatPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DinatConfig base_model_prefix = "dinat" main_input_name = "pixel_values" @@ -641,40 +623,13 @@ class DinatPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -DINAT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`DinatConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DINAT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Dinat Model transformer outputting raw hidden-states without any specific head on top.", - DINAT_START_DOCSTRING, -) +@auto_docstring class DinatModel(DinatPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) requires_backends(self, ["natten"]) @@ -703,14 +658,7 @@ class DinatModel(DinatPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DINAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=DinatModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -758,12 +706,11 @@ class DinatModel(DinatPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Dinat Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - DINAT_START_DOCSTRING, + """ ) class DinatForImageClassification(DinatPreTrainedModel): def __init__(self, config): @@ -782,13 +729,7 @@ class DinatForImageClassification(DinatPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DINAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=DinatImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -852,9 +793,10 @@ class DinatForImageClassification(DinatPreTrainedModel): ) -@add_start_docstrings( - "NAT backbone, to be used with frameworks like DETR and MaskFormer.", - DINAT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + NAT backbone, to be used with frameworks like DETR and MaskFormer. + """ ) class DinatBackbone(DinatPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -879,8 +821,7 @@ class DinatBackbone(DinatPreTrainedModel, BackboneMixin): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(DINAT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -888,9 +829,7 @@ class DinatBackbone(DinatPreTrainedModel, BackboneMixin): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/dinov2/modeling_dinov2.py b/src/transformers/models/dinov2/modeling_dinov2.py index 7ed5a4ec6c..2fb1be118c 100644 --- a/src/transformers/models/dinov2/modeling_dinov2.py +++ b/src/transformers/models/dinov2/modeling_dinov2.py @@ -23,39 +23,16 @@ from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import ( - BackboneOutput, - BaseModelOutput, - BaseModelOutputWithPooling, - ImageClassifierOutput, -) +from ...modeling_outputs import BackboneOutput, BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from ...utils.backbone_utils import BackboneMixin from .configuration_dinov2 import Dinov2Config logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "Dinov2Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/dinov2-base" -_EXPECTED_OUTPUT_SHAPE = [1, 257, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/dinov2-small-imagenet1k-1-layer" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - class Dinov2Embeddings(nn.Module): """ @@ -508,12 +485,8 @@ class Dinov2Encoder(nn.Module): ) +@auto_docstring class Dinov2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Dinov2Config base_model_prefix = "dinov2" main_input_name = "pixel_values" @@ -554,70 +527,7 @@ class Dinov2PreTrainedModel(PreTrainedModel): module.lambda1.data.fill_(self.config.layerscale_value) -DINOV2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Dinov2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DINOV2_BASE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`BitImageProcessor.preprocess`] for details. - - bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for - pre-training. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -DINOV2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`BitImageProcessor.preprocess`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare DINOv2 Model transformer outputting raw hidden-states without any specific head on top.", - DINOV2_START_DOCSTRING, -) +@auto_docstring class Dinov2Model(Dinov2PreTrainedModel): def __init__(self, config: Dinov2Config): super().__init__(config) @@ -642,14 +552,7 @@ class Dinov2Model(Dinov2PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DINOV2_BASE_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -659,6 +562,11 @@ class Dinov2Model(Dinov2PreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for + pre-training. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -700,12 +608,11 @@ class Dinov2Model(Dinov2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Dinov2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - DINOV2_START_DOCSTRING, + """ ) class Dinov2ForImageClassification(Dinov2PreTrainedModel): def __init__(self, config: Dinov2Config) -> None: @@ -722,13 +629,7 @@ class Dinov2ForImageClassification(Dinov2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DINOV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -800,11 +701,10 @@ class Dinov2ForImageClassification(Dinov2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Dinov2 backbone, to be used with frameworks like DETR and MaskFormer. - """, - DINOV2_START_DOCSTRING, + """ ) class Dinov2Backbone(Dinov2PreTrainedModel, BackboneMixin): def __init__(self, config): @@ -823,8 +723,7 @@ class Dinov2Backbone(Dinov2PreTrainedModel, BackboneMixin): def get_input_embeddings(self) -> Dinov2PatchEmbeddings: return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(DINOV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -832,9 +731,7 @@ class Dinov2Backbone(Dinov2PreTrainedModel, BackboneMixin): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/dinov2_with_registers/modeling_dinov2_with_registers.py b/src/transformers/models/dinov2_with_registers/modeling_dinov2_with_registers.py index 4851d16ac8..35e5e02e86 100644 --- a/src/transformers/models/dinov2_with_registers/modeling_dinov2_with_registers.py +++ b/src/transformers/models/dinov2_with_registers/modeling_dinov2_with_registers.py @@ -31,26 +31,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput, BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from ...utils.backbone_utils import BackboneMixin from .configuration_dinov2_with_registers import Dinov2WithRegistersConfig logger = logging.get_logger(__name__) -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/dinov2_with_registers-base" - -# General docstring -_CONFIG_FOR_DOC = "Dinov2WithRegistersConfig" - class Dinov2WithRegistersPatchEmbeddings(nn.Module): """ @@ -516,12 +503,8 @@ class Dinov2WithRegistersEncoder(nn.Module): ) +@auto_docstring class Dinov2WithRegistersPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Dinov2WithRegistersConfig base_model_prefix = "dinov2_with_registers" main_input_name = "pixel_values" @@ -562,51 +545,7 @@ class Dinov2WithRegistersPreTrainedModel(PreTrainedModel): module.lambda1.data.fill_(self.config.layerscale_value) -_EXPECTED_OUTPUT_SHAPE = [1, 257, 768] - - -DINOV2_WITH_REGISTERS_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Dinov2WithRegistersConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DINOV2_WITH_REGISTERS_BASE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`BitImageProcessor.preprocess`] for details. - - bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for - pre-training. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Dinov2WithRegisters Model transformer outputting raw hidden-states without any specific head on top.", - DINOV2_WITH_REGISTERS_START_DOCSTRING, -) +@auto_docstring class Dinov2WithRegistersModel(Dinov2WithRegistersPreTrainedModel): def __init__(self, config: Dinov2WithRegistersConfig): super().__init__(config) @@ -631,14 +570,7 @@ class Dinov2WithRegistersModel(Dinov2WithRegistersPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DINOV2_WITH_REGISTERS_BASE_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -648,6 +580,11 @@ class Dinov2WithRegistersModel(Dinov2WithRegistersPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for + pre-training. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -689,39 +626,11 @@ class Dinov2WithRegistersModel(Dinov2WithRegistersPreTrainedModel): ) -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/dinov2_with_registers-small-imagenet1k-1-layer" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - -DINOV2_WITH_REGISTERS_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`BitImageProcessor.preprocess`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Dinov2WithRegisters Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - DINOV2_WITH_REGISTERS_START_DOCSTRING, + """ ) class Dinov2WithRegistersForImageClassification(Dinov2WithRegistersPreTrainedModel): def __init__(self, config: Dinov2WithRegistersConfig) -> None: @@ -738,13 +647,7 @@ class Dinov2WithRegistersForImageClassification(Dinov2WithRegistersPreTrainedMod # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DINOV2_WITH_REGISTERS_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -817,11 +720,10 @@ class Dinov2WithRegistersForImageClassification(Dinov2WithRegistersPreTrainedMod ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Dinov2WithRegisters backbone, to be used with frameworks like DETR and MaskFormer. - """, - DINOV2_WITH_REGISTERS_START_DOCSTRING, + """ ) class Dinov2WithRegistersBackbone(Dinov2WithRegistersPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -841,8 +743,7 @@ class Dinov2WithRegistersBackbone(Dinov2WithRegistersPreTrainedModel, BackboneMi def get_input_embeddings(self) -> Dinov2WithRegistersPatchEmbeddings: return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(DINOV2_WITH_REGISTERS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -850,14 +751,8 @@ class Dinov2WithRegistersBackbone(Dinov2WithRegistersPreTrainedModel, BackboneMi output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: - Returns: - - Examples: - ```python >>> from transformers import AutoImageProcessor, AutoBackbone diff --git a/src/transformers/models/dinov2_with_registers/modular_dinov2_with_registers.py b/src/transformers/models/dinov2_with_registers/modular_dinov2_with_registers.py index 4ab7521f09..226653cd40 100644 --- a/src/transformers/models/dinov2_with_registers/modular_dinov2_with_registers.py +++ b/src/transformers/models/dinov2_with_registers/modular_dinov2_with_registers.py @@ -413,8 +413,6 @@ class Dinov2WithRegistersBackbone(Dinov2Backbone): return_dict: Optional[bool] = None, ) -> BackboneOutput: """ - Returns: - Examples: ```python diff --git a/src/transformers/models/distilbert/modeling_distilbert.py b/src/transformers/models/distilbert/modeling_distilbert.py index 380543c092..1a24188414 100755 --- a/src/transformers/models/distilbert/modeling_distilbert.py +++ b/src/transformers/models/distilbert/modeling_distilbert.py @@ -47,11 +47,8 @@ from ...pytorch_utils import ( prune_linear_layer, ) from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_distilbert import DistilBertConfig @@ -61,8 +58,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "distilbert-base-uncased" -_CONFIG_FOR_DOC = "DistilBertConfig" # UTILS AND BUILDING BLOCKS OF THE ARCHITECTURE # @@ -577,12 +572,8 @@ class Transformer(nn.Module): # INTERFACE FOR ENCODER AND TASK SPECIFIC MODEL # +@auto_docstring class DistilBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DistilBertConfig load_tf_weights = None base_model_prefix = "distilbert" @@ -611,63 +602,7 @@ class DistilBertPreTrainedModel(PreTrainedModel): ) -DISTILBERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DistilBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DISTILBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare DistilBERT encoder/transformer outputting raw hidden-states without any specific head on top.", - DISTILBERT_START_DOCSTRING, -) +@auto_docstring class DistilBertModel(DistilBertPreTrainedModel): def __init__(self, config: PretrainedConfig): super().__init__(config) @@ -742,12 +677,7 @@ class DistilBertModel(DistilBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.transformer.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -758,6 +688,19 @@ class DistilBertModel(DistilBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[BaseModelOutput, Tuple[torch.Tensor, ...]]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -803,9 +746,10 @@ class DistilBertModel(DistilBertPreTrainedModel): ) -@add_start_docstrings( - """DistilBert Model with a `masked language modeling` head on top.""", - DISTILBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + DistilBert Model with a `masked language modeling` head on top. + """ ) class DistilBertForMaskedLM(DistilBertPreTrainedModel): _tied_weights_keys = ["vocab_projector.weight"] @@ -851,12 +795,7 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel): def set_output_embeddings(self, new_embeddings: nn.Module): self.vocab_projector = new_embeddings - @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -869,6 +808,17 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[MaskedLMOutput, Tuple[torch.Tensor, ...]]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the @@ -907,12 +857,11 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DistilBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - DISTILBERT_START_DOCSTRING, + """ ) class DistilBertForSequenceClassification(DistilBertPreTrainedModel): def __init__(self, config: PretrainedConfig): @@ -948,12 +897,7 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel): """ self.distilbert.resize_position_embeddings(new_num_position_embeddings) - @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1024,13 +968,7 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel): ) -@add_start_docstrings( - """ - DistilBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - DISTILBERT_START_DOCSTRING, -) +@auto_docstring class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): def __init__(self, config: PretrainedConfig): super().__init__(config) @@ -1065,12 +1003,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): """ self.distilbert.resize_position_embeddings(new_num_position_embeddings) - @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1084,14 +1017,17 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[QuestionAnsweringModelOutput, Tuple[torch.Tensor, ...]]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1142,13 +1078,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): ) -@add_start_docstrings( - """ - DistilBert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - DISTILBERT_START_DOCSTRING, -) +@auto_docstring class DistilBertForTokenClassification(DistilBertPreTrainedModel): def __init__(self, config: PretrainedConfig): super().__init__(config) @@ -1181,12 +1111,7 @@ class DistilBertForTokenClassification(DistilBertPreTrainedModel): """ self.distilbert.resize_position_embeddings(new_num_position_embeddings) - @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1236,13 +1161,7 @@ class DistilBertForTokenClassification(DistilBertPreTrainedModel): ) -@add_start_docstrings( - """ - DistilBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and - a softmax) e.g. for RocStories/SWAG tasks. - """, - DISTILBERT_START_DOCSTRING, -) +@auto_docstring class DistilBertForMultipleChoice(DistilBertPreTrainedModel): def __init__(self, config: PretrainedConfig): super().__init__(config) @@ -1275,10 +1194,7 @@ class DistilBertForMultipleChoice(DistilBertPreTrainedModel): """ self.distilbert.resize_position_embeddings(new_num_position_embeddings) - @add_start_docstrings_to_model_forward( - DISTILBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @replace_return_docstrings(output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1291,13 +1207,22 @@ class DistilBertForMultipleChoice(DistilBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[MultipleChoiceModelOutput, Tuple[torch.Tensor, ...]]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) - Returns: - Examples: ```python diff --git a/src/transformers/models/donut/image_processing_donut_fast.py b/src/transformers/models/donut/image_processing_donut_fast.py index be83b5cc5c..56bb290ccc 100644 --- a/src/transformers/models/donut/image_processing_donut_fast.py +++ b/src/transformers/models/donut/image_processing_donut_fast.py @@ -17,8 +17,6 @@ from typing import Optional, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -34,7 +32,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -55,15 +53,8 @@ if is_torchvision_available(): class DonutFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): - do_thumbnail: Optional[bool] - do_align_long_axis: Optional[bool] - do_pad: Optional[bool] - - -@add_start_docstrings( - "Constructs a fast Donut image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, """ + Args: do_thumbnail (`bool`, *optional*, defaults to `self.do_thumbnail`): Whether to resize the image using thumbnail method. do_align_long_axis (`bool`, *optional*, defaults to `self.do_align_long_axis`): @@ -72,8 +63,14 @@ class DonutFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): Whether to pad the image. If `random_padding` is set to `True`, each image is padded with a random amount of padding on each size, up to the largest image size in the batch. Otherwise, all images are padded to the largest image size in the batch. - """, -) + """ + + do_thumbnail: Optional[bool] + do_align_long_axis: Optional[bool] + do_pad: Optional[bool] + + +@auto_docstring class DonutImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN @@ -94,19 +91,7 @@ class DonutImageProcessorFast(BaseImageProcessorFast): kwargs["size"] = size super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - do_thumbnail (`bool`, *optional*, defaults to `self.do_thumbnail`): - Whether to resize the image using thumbnail method. - do_align_long_axis (`bool`, *optional*, defaults to `self.do_align_long_axis`): - Whether to align the long axis of the image with the long axis of `size` by rotating by 90 degrees. - do_pad (`bool`, *optional*, defaults to `self.do_pad`): - Whether to pad the image. If `random_padding` is set to `True`, each image is padded with a random - amount of padding on each size, up to the largest image size in the batch. Otherwise, all images are - padded to the largest image size in the batch. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[DonutFastImageProcessorKwargs]) -> BatchFeature: if "size" in kwargs: size = kwargs.pop("size") diff --git a/src/transformers/models/donut/modeling_donut_swin.py b/src/transformers/models/donut/modeling_donut_swin.py index 43d8f3f107..c1fd629466 100644 --- a/src/transformers/models/donut/modeling_donut_swin.py +++ b/src/transformers/models/donut/modeling_donut_swin.py @@ -29,30 +29,12 @@ from torch import nn from ...activations import ACT2FN from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, meshgrid, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_donut_swin import DonutSwinConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "DonutSwinConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "https://huggingface.co/naver-clova-ix/donut-base" -_EXPECTED_OUTPUT_SHAPE = [1, 49, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "eljandoubi/donut-base-encoder" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - @dataclass # Copied from transformers.models.swin.modeling_swin.SwinEncoderOutput with Swin->DonutSwin @@ -886,13 +868,9 @@ class DonutSwinEncoder(nn.Module): ) +@auto_docstring # Copied from transformers.models.swin.modeling_swin.SwinPreTrainedModel with Swin->DonutSwin,swin->donut class DonutSwinPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DonutSwinConfig base_model_prefix = "donut" main_input_name = "pixel_values" @@ -919,47 +897,15 @@ class DonutSwinPreTrainedModel(PreTrainedModel): module.relative_position_bias_table.data.zero_() -SWIN_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`DonutSwinConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SWIN_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`DonutImageProcessor.__call__`] for details. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Donut Swin Model transformer outputting raw hidden-states without any specific head on top.", - SWIN_START_DOCSTRING, -) +@auto_docstring class DonutSwinModel(DonutSwinPreTrainedModel): def __init__(self, config, add_pooling_layer=True, use_mask_token=False): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config self.num_layers = len(config.depths) @@ -984,14 +930,7 @@ class DonutSwinModel(DonutSwinPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SWIN_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=DonutSwinModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1056,8 +995,8 @@ class DonutSwinModel(DonutSwinPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DonutSwin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. @@ -1068,8 +1007,7 @@ class DonutSwinModel(DonutSwinPreTrainedModel): position embeddings to the higher resolution. - """, - SWIN_START_DOCSTRING, + """ ) # Copied from transformers.models.swin.modeling_swin.SwinForImageClassification with Swin->DonutSwin,swin->donut class DonutSwinForImageClassification(DonutSwinPreTrainedModel): @@ -1087,13 +1025,7 @@ class DonutSwinForImageClassification(DonutSwinPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWIN_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=DonutSwinImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, diff --git a/src/transformers/models/dpr/modeling_dpr.py b/src/transformers/models/dpr/modeling_dpr.py index 3ff4aa1152..c707d4fcfe 100644 --- a/src/transformers/models/dpr/modeling_dpr.py +++ b/src/transformers/models/dpr/modeling_dpr.py @@ -24,10 +24,8 @@ from ...modeling_outputs import BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from ..bert.modeling_bert import BertModel from .configuration_dpr import DPRConfig @@ -35,9 +33,6 @@ from .configuration_dpr import DPRConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "DPRConfig" -_CHECKPOINT_FOR_DOC = "facebook/dpr-ctx_encoder-single-nq-base" - ########## # Outputs @@ -133,6 +128,7 @@ class DPRReaderOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor, ...]] = None +@auto_docstring class DPRPreTrainedModel(PreTrainedModel): _supports_sdpa = True @@ -309,24 +305,31 @@ class DPRPretrainedReader(DPRPreTrainedModel): ############### -DPR_START_DOCSTRING = r""" +@auto_docstring( + custom_intro=""" + The bare DPRContextEncoder transformer outputting pooler outputs as context representations. + """ +) +class DPRContextEncoder(DPRPretrainedContextEncoder): + def __init__(self, config: DPRConfig): + super().__init__(config) + self.config = config + self.ctx_encoder = DPREncoder(config) + # Initialize weights and apply final processing + self.post_init() - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`DPRConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DPR_ENCODERS_INPUTS_DOCSTRING = r""" - Args: + @auto_docstring + def forward( + self, + input_ids: Optional[Tensor] = None, + attention_mask: Optional[Tensor] = None, + token_type_ids: Optional[Tensor] = None, + inputs_embeds: Optional[Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[DPRContextEncoderOutput, Tuple[Tensor, ...]]: + r""" input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. To match pretraining, DPR input sequence should be formatted with [CLS] and [SEP] tokens as follows: @@ -352,98 +355,6 @@ DPR_ENCODERS_INPUTS_DOCSTRING = r""" [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -DPR_READER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`Tuple[torch.LongTensor]` of shapes `(n_passages, sequence_length)`): - Indices of input sequence tokens in the vocabulary. It has to be a sequence triplet with 1) the question - and 2) the passages titles and 3) the passages texts To match pretraining, DPR `input_ids` sequence should - be formatted with [CLS] and [SEP] with the format: - - `[CLS] [SEP] [SEP] ` - - DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right - rather than the left. - - Indices can be obtained using [`DPRReaderTokenizer`]. See this class documentation for more details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(n_passages, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - inputs_embeds (`torch.FloatTensor` of shape `(n_passages, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare DPRContextEncoder transformer outputting pooler outputs as context representations.", - DPR_START_DOCSTRING, -) -class DPRContextEncoder(DPRPretrainedContextEncoder): - def __init__(self, config: DPRConfig): - super().__init__(config) - self.config = config - self.ctx_encoder = DPREncoder(config) - # Initialize weights and apply final processing - self.post_init() - - @add_start_docstrings_to_model_forward(DPR_ENCODERS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DPRContextEncoderOutput, config_class=_CONFIG_FOR_DOC) - def forward( - self, - input_ids: Optional[Tensor] = None, - attention_mask: Optional[Tensor] = None, - token_type_ids: Optional[Tensor] = None, - inputs_embeds: Optional[Tensor] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[DPRContextEncoderOutput, Tuple[Tensor, ...]]: - r""" - Return: Examples: @@ -499,9 +410,10 @@ class DPRContextEncoder(DPRPretrainedContextEncoder): ) -@add_start_docstrings( - "The bare DPRQuestionEncoder transformer outputting pooler outputs as question representations.", - DPR_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare DPRQuestionEncoder transformer outputting pooler outputs as question representations. + """ ) class DPRQuestionEncoder(DPRPretrainedQuestionEncoder): def __init__(self, config: DPRConfig): @@ -511,8 +423,7 @@ class DPRQuestionEncoder(DPRPretrainedQuestionEncoder): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DPR_ENCODERS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DPRQuestionEncoderOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[Tensor] = None, @@ -524,7 +435,31 @@ class DPRQuestionEncoder(DPRPretrainedQuestionEncoder): return_dict: Optional[bool] = None, ) -> Union[DPRQuestionEncoderOutput, Tuple[Tensor, ...]]: r""" - Return: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. To match pretraining, DPR input sequence should be + formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs (for a pair title+text for example): + + ``` + tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + ``` + + (b) For single sequences (for a question for example): + + ``` + tokens: [CLS] the dog is hairy . [SEP] + token_type_ids: 0 0 0 0 0 0 0 + ``` + + DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right + rather than the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) Examples: @@ -581,9 +516,10 @@ class DPRQuestionEncoder(DPRPretrainedQuestionEncoder): ) -@add_start_docstrings( - "The bare DPRReader transformer outputting span predictions.", - DPR_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare DPRReader transformer outputting span predictions. + """ ) class DPRReader(DPRPretrainedReader): def __init__(self, config: DPRConfig): @@ -593,8 +529,7 @@ class DPRReader(DPRPretrainedReader): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DPR_READER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DPRReaderOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[Tensor] = None, @@ -605,7 +540,23 @@ class DPRReader(DPRPretrainedReader): return_dict: Optional[bool] = None, ) -> Union[DPRReaderOutput, Tuple[Tensor, ...]]: r""" - Return: + input_ids (`Tuple[torch.LongTensor]` of shapes `(n_passages, sequence_length)`): + Indices of input sequence tokens in the vocabulary. It has to be a sequence triplet with 1) the question + and 2) the passages titles and 3) the passages texts To match pretraining, DPR `input_ids` sequence should + be formatted with [CLS] and [SEP] with the format: + + `[CLS] [SEP] [SEP] ` + + DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right + rather than the left. + + Indices can be obtained using [`DPRReaderTokenizer`]. See this class documentation for more details. + + [What are input IDs?](../glossary#input-ids) + inputs_embeds (`torch.FloatTensor` of shape `(n_passages, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. Examples: diff --git a/src/transformers/models/dpt/modeling_dpt.py b/src/transformers/models/dpt/modeling_dpt.py index 0dc1dcf2f8..d772ddb85b 100755 --- a/src/transformers/models/dpt/modeling_dpt.py +++ b/src/transformers/models/dpt/modeling_dpt.py @@ -29,29 +29,16 @@ from torch import nn from torch.nn import CrossEntropyLoss from ...activations import ACT2FN -from ...file_utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) from ...modeling_outputs import BaseModelOutput, DepthEstimatorOutput, SemanticSegmenterOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ModelOutput, logging, torch_int +from ...utils import ModelOutput, auto_docstring, logging, torch_int from ...utils.backbone_utils import load_backbone from .configuration_dpt import DPTConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "DPTConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "Intel/dpt-large" -_EXPECTED_OUTPUT_SHAPE = [1, 577, 1024] - @dataclass class BaseModelOutputWithIntermediateActivations(ModelOutput): @@ -831,12 +818,8 @@ class DPTFeatureFusionLayer(nn.Module): return hidden_state +@auto_docstring class DPTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = DPTConfig base_model_prefix = "dpt" main_input_name = "pixel_values" @@ -860,46 +843,13 @@ class DPTPreTrainedModel(PreTrainedModel): module.position_embeddings.data.zero_() -DPT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ViTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -DPT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare DPT Model transformer outputting raw hidden-states without any specific head on top.", - DPT_START_DOCSTRING, -) +@auto_docstring class DPTModel(DPTPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -930,14 +880,7 @@ class DPTModel(DPTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(DPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndIntermediateActivations, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1098,11 +1041,10 @@ class DPTDepthEstimationHead(nn.Module): return predicted_depth -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" DPT Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. - """, - DPT_START_DOCSTRING, + """ ) class DPTForDepthEstimation(DPTPreTrainedModel): def __init__(self, config): @@ -1123,8 +1065,7 @@ class DPTForDepthEstimation(DPTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1138,8 +1079,6 @@ class DPTForDepthEstimation(DPTPreTrainedModel): labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth depth estimation maps for computing the loss. - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, DPTForDepthEstimation @@ -1282,12 +1221,7 @@ class DPTAuxiliaryHead(nn.Module): return logits -@add_start_docstrings( - """ - DPT Model with a semantic segmentation head on top e.g. for ADE20k, CityScapes. - """, - DPT_START_DOCSTRING, -) +@auto_docstring class DPTForSemanticSegmentation(DPTPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1304,8 +1238,7 @@ class DPTForSemanticSegmentation(DPTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(DPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1320,8 +1253,6 @@ class DPTForSemanticSegmentation(DPTPreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, DPTForSemanticSegmentation diff --git a/src/transformers/models/efficientnet/image_processing_efficientnet_fast.py b/src/transformers/models/efficientnet/image_processing_efficientnet_fast.py index fb63956401..ef956e5000 100644 --- a/src/transformers/models/efficientnet/image_processing_efficientnet_fast.py +++ b/src/transformers/models/efficientnet/image_processing_efficientnet_fast.py @@ -18,8 +18,6 @@ from functools import lru_cache from typing import Optional, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -35,7 +33,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -53,14 +51,19 @@ if is_torchvision_available(): class EfficientNetFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + Args: + rescale_offset (`bool`, *optional*, defaults to `self.rescale_offset`): + Whether to rescale the image between [-max_range/2, scale_range/2] instead of [0, scale_range]. + include_top (`bool`, *optional*, defaults to `self.include_top`): + Normalize the image again with the standard deviation only for image classification if set to True. + """ + rescale_offset: bool include_top: bool -@add_start_docstrings( - "Constructs a fast EfficientNet image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class EfficientNetImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.NEAREST image_mean = IMAGENET_STANDARD_MEAN @@ -210,15 +213,7 @@ class EfficientNetImageProcessorFast(BaseImageProcessorFast): return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - rescale_offset (`bool`, *optional*, defaults to `self.rescale_offset`): - Whether to rescale the image between [-max_range/2, scale_range/2] instead of [0, scale_range]. - include_top (`bool`, *optional*, defaults to `self.include_top`): - Normalize the image again with the standard deviation only for image classification if set to True. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[EfficientNetFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/efficientnet/modeling_efficientnet.py b/src/transformers/models/efficientnet/modeling_efficientnet.py index 7943b5d117..d629cf622c 100644 --- a/src/transformers/models/efficientnet/modeling_efficientnet.py +++ b/src/transformers/models/efficientnet/modeling_efficientnet.py @@ -29,53 +29,12 @@ from ...modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import auto_docstring, logging from .configuration_efficientnet import EfficientNetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "EfficientNetConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "google/efficientnet-b7" -_EXPECTED_OUTPUT_SHAPE = [1, 768, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "google/efficientnet-b7" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - - -EFFICIENTNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`EfficientNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -EFFICIENTNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - def round_filters(config: EfficientNetConfig, num_channels: int): r""" @@ -471,12 +430,8 @@ class EfficientNetEncoder(nn.Module): ) +@auto_docstring class EfficientNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = EfficientNetConfig base_model_prefix = "efficientnet" main_input_name = "pixel_values" @@ -495,10 +450,7 @@ class EfficientNetPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -@add_start_docstrings( - "The bare EfficientNet model outputting raw features without any specific head on top.", - EFFICIENTNET_START_DOCSTRING, -) +@auto_docstring class EfficientNetModel(EfficientNetPreTrainedModel): def __init__(self, config: EfficientNetConfig): super().__init__(config) @@ -517,14 +469,7 @@ class EfficientNetModel(EfficientNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(EFFICIENTNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -562,12 +507,11 @@ class EfficientNetModel(EfficientNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" EfficientNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - EFFICIENTNET_START_DOCSTRING, + """ ) class EfficientNetForImageClassification(EfficientNetPreTrainedModel): def __init__(self, config): @@ -582,13 +526,7 @@ class EfficientNetForImageClassification(EfficientNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(EFFICIENTNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, diff --git a/src/transformers/models/electra/modeling_electra.py b/src/transformers/models/electra/modeling_electra.py index 08cc3e530d..0ebfa3d479 100644 --- a/src/transformers/models/electra/modeling_electra.py +++ b/src/transformers/models/electra/modeling_electra.py @@ -40,20 +40,14 @@ from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_electra import ElectraConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/electra-small-discriminator" -_CONFIG_FOR_DOC = "ElectraConfig" - def load_tf_weights_in_electra(model, config, tf_checkpoint_path, discriminator_or_generator="discriminator"): """Load tf checkpoints in a pytorch model.""" @@ -659,12 +653,8 @@ class ElectraGeneratorPredictions(nn.Module): return hidden_states +@auto_docstring class ElectraPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ElectraConfig load_tf_weights = load_tf_weights_in_electra base_model_prefix = "electra" @@ -716,90 +706,7 @@ class ElectraForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -ELECTRA_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ElectraConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ELECTRA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - encoder_hidden_states (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Electra Model transformer outputting raw hidden-states without any specific head on top. Identical to " - "the BERT model except that it uses an additional linear layer between the embedding layer and the encoder if the " - "hidden size and embedding size are different. " - "" - "Both the generator and discriminator checkpoints may be loaded into this model.", - ELECTRA_START_DOCSTRING, -) +@auto_docstring class ElectraModel(ElectraPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -827,12 +734,7 @@ class ElectraModel(ElectraPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1046,12 +948,11 @@ class ElectraSequenceSummary(nn.Module): return output -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ELECTRA_START_DOCSTRING, + """ ) class ElectraForSequenceClassification(ElectraPreTrainedModel): def __init__(self, config): @@ -1064,14 +965,7 @@ class ElectraForSequenceClassification(ElectraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="bhadresh-savani/electra-base-emotion", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'joy'", - expected_loss=0.06, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1143,13 +1037,12 @@ class ElectraForSequenceClassification(ElectraPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Electra model with a binary classification head on top as used during pretraining for identifying generated tokens. It is recommended to load the discriminator checkpoint into that model. - """, - ELECTRA_START_DOCSTRING, + """ ) class ElectraForPreTraining(ElectraPreTrainedModel): def __init__(self, config): @@ -1160,8 +1053,7 @@ class ElectraForPreTraining(ElectraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=ElectraForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1183,8 +1075,6 @@ class ElectraForPreTraining(ElectraPreTrainedModel): - 0 indicates the token is an original token, - 1 indicates the token was replaced. - Returns: - Examples: ```python @@ -1248,14 +1138,13 @@ class ElectraForPreTraining(ElectraPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Electra model with a language modeling head on top. Even though both the discriminator and generator may be loaded into this model, the generator is the only model of the two to have been trained for the masked language modeling task. - """, - ELECTRA_START_DOCSTRING, + """ ) class ElectraForMaskedLM(ElectraPreTrainedModel): _tied_weights_keys = ["generator_lm_head.weight"] @@ -1276,15 +1165,7 @@ class ElectraForMaskedLM(ElectraPreTrainedModel): def set_output_embeddings(self, word_embeddings): self.generator_lm_head = word_embeddings - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/electra-small-generator", - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="[MASK]", - expected_output="'paris'", - expected_loss=1.22, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1340,13 +1221,12 @@ class ElectraForMaskedLM(ElectraPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Electra model with a token classification head on top. Both the discriminator and generator may be loaded into this model. - """, - ELECTRA_START_DOCSTRING, + """ ) class ElectraForTokenClassification(ElectraPreTrainedModel): def __init__(self, config): @@ -1362,14 +1242,7 @@ class ElectraForTokenClassification(ElectraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="bhadresh-savani/electra-base-discriminator-finetuned-conll03-english", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="['B-LOC', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'B-LOC', 'O', 'B-LOC', 'I-LOC']", - expected_loss=0.11, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1422,13 +1295,7 @@ class ElectraForTokenClassification(ElectraPreTrainedModel): ) -@add_start_docstrings( - """ - ELECTRA Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - ELECTRA_START_DOCSTRING, -) +@auto_docstring class ElectraForQuestionAnswering(ElectraPreTrainedModel): config_class = ElectraConfig base_model_prefix = "electra" @@ -1443,16 +1310,7 @@ class ElectraForQuestionAnswering(ElectraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="bhadresh-savani/electra-base-squad2", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - qa_target_start_index=11, - qa_target_end_index=12, - expected_output="'a nice puppet'", - expected_loss=2.64, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1467,16 +1325,6 @@ class ElectraForQuestionAnswering(ElectraPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict discriminator_hidden_states = self.electra( @@ -1530,13 +1378,7 @@ class ElectraForQuestionAnswering(ElectraPreTrainedModel): ) -@add_start_docstrings( - """ - ELECTRA Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - ELECTRA_START_DOCSTRING, -) +@auto_docstring class ElectraForMultipleChoice(ElectraPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1548,12 +1390,7 @@ class ElectraForMultipleChoice(ElectraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1568,6 +1405,30 @@ class ElectraForMultipleChoice(ElectraPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1621,8 +1482,10 @@ class ElectraForMultipleChoice(ElectraPreTrainedModel): ) -@add_start_docstrings( - """ELECTRA Model with a `language modeling` head on top for CLM fine-tuning.""", ELECTRA_START_DOCSTRING +@auto_docstring( + custom_intro=""" + ELECTRA Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class ElectraForCausalLM(ElectraPreTrainedModel, GenerationMixin): _tied_weights_keys = ["generator_lm_head.weight"] @@ -1645,8 +1508,7 @@ class ElectraForCausalLM(ElectraPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.generator_lm_head = new_embeddings - @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1666,31 +1528,10 @@ class ElectraForCausalLM(ElectraPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 761e40a2fe..83136bfba2 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -39,15 +39,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_emu3 import Emu3Config, Emu3TextConfig, Emu3VQVAEConfig @@ -60,9 +52,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Emu3Config" - - @use_kernel_forward_from_hub("RMSNorm") class Emu3RMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -979,29 +968,12 @@ class Emu3VQVAEDecoder(nn.Module): return hidden_states -EMU3_VQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Emu3VQVAEConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens. +@auto_docstring( + custom_intro=""" + The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens. This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv Taigman](https://arxiv.org/abs/2203.13131). - """, - EMU3_VQ_START_DOCSTRING, + """ ) class Emu3VQVAE(PreTrainedModel): config_class = Emu3VQVAEConfig @@ -1168,27 +1140,7 @@ class Emu3ImageVocabularyMapping: return img_tokens.to(device) -EMU3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Emu3Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare emu3 Model outputting raw hidden-states without any specific head on top.", - EMU3_START_DOCSTRING, -) +@auto_docstring class Emu3PreTrainedModel(PreTrainedModel): config_class = Emu3Config base_model_prefix = "model" @@ -1254,89 +1206,8 @@ class Emu3RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -EMU3_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Has to be an instance of [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - The model will output the same cache type that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Emu3Text Model outputting raw hidden-states without any specific head on top.", - EMU3_START_DOCSTRING, -) +@auto_docstring class Emu3TextModel(Emu3PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Emu3TextDecoderLayer`] - - Args: - config: Emu3TextConfig - """ - def __init__(self, config: Emu3Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1360,7 +1231,7 @@ class Emu3TextModel(Emu3PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_TEXT_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1582,6 +1453,7 @@ class Emu3TextModel(Emu3PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Emu3ForCausalLM(Emu3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -1616,8 +1488,7 @@ class Emu3ForCausalLM(Emu3PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class="Emu3TextConfig") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1634,19 +1505,10 @@ class Emu3ForCausalLM(Emu3PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1701,85 +1563,6 @@ class Emu3ForCausalLM(Emu3PreTrainedModel, GenerationMixin): ) -EMU3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, max_num_images, max_num_tiles, channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses - [`Emu3ImageProcessor`] for processing images). - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): - The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using - [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses - [`Emu3ImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Has to be an instance of [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class Emu3Model(Emu3PreTrainedModel): _checkpoint_conversion_mapping = {"text_model.model": "text_model"} _supports_static_cache = False # `get_image_tokens()`, called when `pixel_values` is passed, is not compileable @@ -1839,7 +1622,7 @@ class Emu3Model(Emu3PreTrainedModel): return image @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1856,6 +1639,12 @@ class Emu3Model(Emu3PreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): + The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using + [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses + [`Emu3ImageProcessor`] for processing images). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1928,8 +1717,7 @@ class Emu3ForConditionalGeneration(Emu3PreTrainedModel, GenerationMixin): return self.model.vqmodel @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1949,19 +1737,14 @@ class Emu3ForConditionalGeneration(Emu3PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): + The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using + [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses + [`Emu3ImageProcessor`] for processing images). + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index 971d94e29d..b1c27f961b 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -26,35 +26,16 @@ import torch.utils.checkpoint from ...cache_utils import Cache from ...generation import GenerationMixin from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - CausalLMOutputWithPast, -) +from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) -from ..chameleon.modeling_chameleon import ( - ChameleonPreTrainedModel, - ChameleonVQVAEEncoderConvDownsample, -) -from ..llama.modeling_llama import ( - KwargsForCausalLM, - LlamaDecoderLayer, - LlamaForCausalLM, - LlamaModel, -) +from ...utils import auto_docstring, can_return_tuple, logging +from ..chameleon.modeling_chameleon import ChameleonPreTrainedModel, ChameleonVQVAEEncoderConvDownsample +from ..llama.modeling_llama import KwargsForCausalLM, LlamaDecoderLayer, LlamaForCausalLM, LlamaModel from ..siglip.modeling_siglip import SiglipAttention from .configuration_emu3 import Emu3Config, Emu3TextConfig, Emu3VQVAEConfig -_CONFIG_FOR_DOC = "Emu3Config" -_CHECKPOINT_FOR_DOC = "BAAI/Emu3-Chat-hf" - logger = logging.get_logger(__name__) @@ -708,29 +689,12 @@ class Emu3VQVAEDecoder(nn.Module): return hidden_states -EMU3_VQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Emu3VQVAEConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens. +@auto_docstring( + custom_intro=""" + The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens. This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv Taigman](https://arxiv.org/abs/2203.13131). - """, - EMU3_VQ_START_DOCSTRING, + """ ) class Emu3VQVAE(PreTrainedModel): config_class = Emu3VQVAEConfig @@ -918,156 +882,6 @@ class Emu3PreTrainedModel(ChameleonPreTrainedModel, Emu3VQVAE): module.weight.data.fill_(1.0) -EMU3_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Has to be an instance of [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - The model will output the same cache type that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -EMU3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, max_num_images, max_num_tiles, channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses - [`Emu3ImageProcessor`] for processing images). - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): - The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using - [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses - [`Emu3ImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Has to be an instance of [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class Emu3TextModel(LlamaModel, Emu3PreTrainedModel): def __init__(self, config: Emu3Config): super().__init__(config) @@ -1075,11 +889,6 @@ class Emu3TextModel(LlamaModel, Emu3PreTrainedModel): [Emu3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] ) - @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_TEXT_INPUTS_DOCSTRING) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - class Emu3ForCausalLM(LlamaForCausalLM, Emu3PreTrainedModel, GenerationMixin): config_class = Emu3TextConfig @@ -1088,25 +897,8 @@ class Emu3ForCausalLM(LlamaForCausalLM, Emu3PreTrainedModel, GenerationMixin): super().__init__(config) self.model = Emu3TextModel(config) - @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class="Emu3TextConfig") def forward(**super_kwargs): r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: - Example: ```python @@ -1185,7 +977,7 @@ class Emu3Model(Emu3PreTrainedModel): return image @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1202,6 +994,12 @@ class Emu3Model(Emu3PreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): + The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using + [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses + [`Emu3ImageProcessor`] for processing images). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1274,8 +1072,7 @@ class Emu3ForConditionalGeneration(Emu3PreTrainedModel, GenerationMixin): return self.model.vqmodel @can_return_tuple - @add_start_docstrings_to_model_forward(EMU3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1295,19 +1092,14 @@ class Emu3ForConditionalGeneration(Emu3PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`): + The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using + [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses + [`Emu3ImageProcessor`] for processing images). + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/encodec/modeling_encodec.py b/src/transformers/models/encodec/modeling_encodec.py index ba699d745e..cc7b3bc72f 100644 --- a/src/transformers/models/encodec/modeling_encodec.py +++ b/src/transformers/models/encodec/modeling_encodec.py @@ -25,10 +25,8 @@ from torch import nn from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_encodec import EncodecConfig @@ -37,7 +35,6 @@ logger = logging.get_logger(__name__) # General docstring -_CONFIG_FOR_DOC = "EncodecConfig" @dataclass @@ -447,12 +444,8 @@ class EncodecResidualVectorQuantizer(nn.Module): return quantized_out +@auto_docstring class EncodecPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = EncodecConfig base_model_prefix = "encodec" main_input_name = "input_values" @@ -483,59 +476,10 @@ class EncodecPreTrainedModel(PreTrainedModel): nn.init.constant_(param, 0.0) -ENCODEC_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`EncodecConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -ENCODEC_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): - Raw audio input converted to Float and padded to the appropriate length in order to be encoded using chunks - of length self.chunk_length and a stride of `config.chunk_stride`. - padding_mask (`torch.BoolTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): - Mask to avoid computing scaling factors on padding token indices (can we avoid computing conv on these+). - Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - - - `padding_mask` should always be passed, unless the input was truncated or not padded. This is because in - order to process tensors effectively, the input audio should be padded so that `input_length % stride = - step` with `step = chunk_length-stride`. This ensures that all chunks are of the same shape - - - - bandwidth (`float`, *optional*): - The target bandwidth. Must be one of `config.target_bandwidths`. If `None`, uses the smallest possible - bandwidth. bandwidth is represented as a thousandth of what it is, e.g. 6kbps bandwidth is represented as - `bandwidth == 6.0` - audio_codes (`torch.LongTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*): - Discret code embeddings computed using `model.encode`. - audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*): - Scaling factor for each `audio_codes` input. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The EnCodec neural audio codec model.", - ENCODEC_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The EnCodec neural audio codec model. + """ ) class EncodecModel(EncodecPreTrainedModel): def __init__(self, config: EncodecConfig): @@ -761,8 +705,7 @@ class EncodecModel(EncodecPreTrainedModel): return (audio_values,) return EncodecDecoderOutput(audio_values) - @add_start_docstrings_to_model_forward(ENCODEC_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=EncodecOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: torch.Tensor, @@ -773,7 +716,31 @@ class EncodecModel(EncodecPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor, torch.Tensor], EncodecOutput]: r""" - Returns: + input_values (`torch.FloatTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): + Raw audio input converted to Float and padded to the appropriate length in order to be encoded using chunks + of length self.chunk_length and a stride of `config.chunk_stride`. + padding_mask (`torch.BoolTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): + Mask to avoid computing scaling factors on padding token indices (can we avoid computing conv on these+). + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + + + `padding_mask` should always be passed, unless the input was truncated or not padded. This is because in + order to process tensors effectively, the input audio should be padded so that `input_length % stride = + step` with `step = chunk_length-stride`. This ensures that all chunks are of the same shape + + + bandwidth (`float`, *optional*): + The target bandwidth. Must be one of `config.target_bandwidths`. If `None`, uses the smallest possible + bandwidth. bandwidth is represented as a thousandth of what it is, e.g. 6kbps bandwidth is represented as + `bandwidth == 6.0` + audio_codes (`torch.LongTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*): + Discret code embeddings computed using `model.encode`. + audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*): + Scaling factor for each `audio_codes` input. Examples: diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index 1ed7a2a5ce..8aaced7398 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -29,7 +29,7 @@ from ...configuration_utils import PretrainedConfig from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutput, Seq2SeqLMOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel, AutoModelForCausalLM from .configuration_encoder_decoder import EncoderDecoderConfig @@ -37,7 +37,6 @@ from .configuration_encoder_decoder import EncoderDecoderConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "EncoderDecoderConfig" DEPRECATION_WARNING = ( "Version v4.12.0 introduces a better way to train encoder-decoder models by computing the loss inside the" @@ -46,107 +45,6 @@ DEPRECATION_WARNING = ( " labels, no need to pass them yourself anymore." ) -ENCODER_DECODER_START_DOCSTRING = r""" - This class can be used to initialize a sequence-to-sequence model with any pretrained autoencoding model as the - encoder and any pretrained autoregressive model as the decoder. The encoder is loaded via - [`~AutoModel.from_pretrained`] function and the decoder is loaded via [`~AutoModelForCausalLM.from_pretrained`] - function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream - generative task, like summarization. - - The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation - tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation - Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi - Zhou, Wei Li, Peter J. Liu. - - After such an Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other models - (see the examples for more information). - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`EncoderDecoderConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ENCODER_DECODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the - right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`. - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - encoder_outputs (`tuple(torch.FloatTensor)`, *optional*): - This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor - of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the - decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. This is useful if you want more control over how to convert `decoder_input_ids` indices - into associated vectors than the model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, - ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - If set to `True`, the model will return a [`~utils.Seq2SeqLMOutput`] instead of a plain tuple. - kwargs (*optional*): Remaining dictionary of keyword arguments. Keyword arguments come in two flavors: - - - Without a prefix which will be input as `**encoder_kwargs` for the encoder forward function. - - With a *decoder_* prefix which will be input as `**decoder_kwargs` for the decoder forward function. -""" - def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): """ @@ -166,7 +64,7 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start return shifted_input_ids -@add_start_docstrings(ENCODER_DECODER_START_DOCSTRING) +@auto_docstring class EncoderDecoderModel(PreTrainedModel, GenerationMixin): r""" [`EncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with one @@ -189,6 +87,12 @@ class EncoderDecoderModel(PreTrainedModel, GenerationMixin): encoder: Optional[PreTrainedModel] = None, decoder: Optional[PreTrainedModel] = None, ): + r""" + encoder (`PreTrainedModel`, *optional*): + The encoder model to use. + decoder (`PreTrainedModel`, *optional*): + The decoder model to use. + """ if config is None and (encoder is None or decoder is None): raise ValueError("Either a configuration or an encoder and a decoder has to be provided.") if config is None: @@ -541,8 +445,7 @@ class EncoderDecoderModel(PreTrainedModel, GenerationMixin): config = EncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config, **kwargs) return cls(encoder=encoder, decoder=decoder, config=config) - @add_start_docstrings_to_model_forward(ENCODER_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -561,7 +464,30 @@ class EncoderDecoderModel(PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the + right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`. + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert `decoder_input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, + ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Examples: diff --git a/src/transformers/models/ernie/modeling_ernie.py b/src/transformers/models/ernie/modeling_ernie.py index 221dd57485..c07627fa79 100644 --- a/src/transformers/models/ernie/modeling_ernie.py +++ b/src/transformers/models/ernie/modeling_ernie.py @@ -39,22 +39,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_ernie import ErnieConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "nghuyong/ernie-1.0-base-zh" -_CONFIG_FOR_DOC = "ErnieConfig" - class ErnieEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings.""" @@ -644,12 +634,8 @@ class ErniePreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class ErniePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ErnieConfig base_model_prefix = "ernie" supports_gradient_checkpointing = True @@ -706,84 +692,8 @@ class ErnieForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -ERNIE_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ErnieConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ERNIE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - task_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Task type embedding is a special embedding to represent the characteristic of different tasks, such as - word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We - assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, - config.task_type_vocab_size-1] - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Ernie Model transformer outputting raw hidden-states without any specific head on top.", - ERNIE_START_DOCSTRING, -) -class ErnieModel(ErniePreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, @@ -791,11 +701,15 @@ class ErnieModel(ErniePreTrainedModel): To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and - `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. """ - +) +class ErnieModel(ErniePreTrainedModel): # Copied from transformers.models.clap.modeling_clap.ClapTextModel.__init__ with ClapText->Ernie def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -824,12 +738,7 @@ class ErnieModel(ErniePreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -848,24 +757,11 @@ class ErnieModel(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -963,12 +859,11 @@ class ErnieModel(ErniePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Ernie Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. - """, - ERNIE_START_DOCSTRING, + """ ) class ErnieForPreTraining(ErniePreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -992,8 +887,7 @@ class ErnieForPreTraining(ErniePreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=ErnieForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1010,20 +904,21 @@ class ErnieForPreTraining(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], ErnieForPreTrainingOutput]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), - the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for computing the next sequence prediction (classification) loss. Input should be a sequence - pair (see `input_ids` docstring) Indices should be in `[0, 1]`: + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence + pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - - 0 indicates sequence B is a continuation of sequence A, - - 1 indicates sequence B is a random sequence. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. - - Returns: + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. Example: @@ -1079,8 +974,10 @@ class ErnieForPreTraining(ErniePreTrainedModel): ) -@add_start_docstrings( - """Ernie Model with a `language modeling` head on top for CLM fine-tuning.""", ERNIE_START_DOCSTRING +@auto_docstring( + custom_intro=""" + Ernie Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class ErnieForCausalLM(ErniePreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1107,12 +1004,7 @@ class ErnieForCausalLM(ErniePreTrainedModel, GenerationMixin): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1133,28 +1025,15 @@ class ErnieForCausalLM(ErniePreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -1212,7 +1091,7 @@ class ErnieForCausalLM(ErniePreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings("""Ernie Model with a `language modeling` head on top.""", ERNIE_START_DOCSTRING) +@auto_docstring class ErnieForMaskedLM(ErniePreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1241,14 +1120,7 @@ class ErnieForMaskedLM(ErniePreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'paris'", - expected_loss=0.88, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1266,6 +1138,11 @@ class ErnieForMaskedLM(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the @@ -1334,9 +1211,10 @@ class ErnieForMaskedLM(ErniePreTrainedModel): return False -@add_start_docstrings( - """Ernie Model with a `next sentence prediction (classification)` head on top.""", - ERNIE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Ernie Model with a `next sentence prediction (classification)` head on top. + """ ) class ErnieForNextSentencePrediction(ErniePreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForNextSentencePrediction.__init__ with Bert->Ernie,bert->ernie @@ -1349,8 +1227,7 @@ class ErnieForNextSentencePrediction(ErniePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=NextSentencePredictorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1367,6 +1244,11 @@ class ErnieForNextSentencePrediction(ErniePreTrainedModel): **kwargs, ) -> Union[Tuple[torch.Tensor], NextSentencePredictorOutput]: r""" + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring). Indices should be in `[0, 1]`: @@ -1374,8 +1256,6 @@ class ErnieForNextSentencePrediction(ErniePreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - Returns: - Example: ```python @@ -1439,12 +1319,11 @@ class ErnieForNextSentencePrediction(ErniePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Ernie Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ERNIE_START_DOCSTRING, + """ ) class ErnieForSequenceClassification(ErniePreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForSequenceClassification.__init__ with Bert->Ernie,bert->ernie @@ -1463,7 +1342,7 @@ class ErnieForSequenceClassification(ErniePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1479,6 +1358,11 @@ class ErnieForSequenceClassification(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1538,13 +1422,7 @@ class ErnieForSequenceClassification(ErniePreTrainedModel): ) -@add_start_docstrings( - """ - Ernie Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - ERNIE_START_DOCSTRING, -) +@auto_docstring class ErnieForMultipleChoice(ErniePreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForMultipleChoice.__init__ with Bert->Ernie,bert->ernie def __init__(self, config): @@ -1560,12 +1438,7 @@ class ErnieForMultipleChoice(ErniePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1581,6 +1454,35 @@ class ErnieForMultipleChoice(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + task_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1635,13 +1537,7 @@ class ErnieForMultipleChoice(ErniePreTrainedModel): ) -@add_start_docstrings( - """ - Ernie Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - ERNIE_START_DOCSTRING, -) +@auto_docstring class ErnieForTokenClassification(ErniePreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForTokenClassification.__init__ with Bert->Ernie,bert->ernie def __init__(self, config): @@ -1658,7 +1554,7 @@ class ErnieForTokenClassification(ErniePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1674,6 +1570,11 @@ class ErnieForTokenClassification(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1714,13 +1615,7 @@ class ErnieForTokenClassification(ErniePreTrainedModel): ) -@add_start_docstrings( - """ - Ernie Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - ERNIE_START_DOCSTRING, -) +@auto_docstring class ErnieForQuestionAnswering(ErniePreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForQuestionAnswering.__init__ with Bert->Ernie,bert->ernie def __init__(self, config): @@ -1733,7 +1628,7 @@ class ErnieForQuestionAnswering(ErniePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ERNIE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1750,14 +1645,11 @@ class ErnieForQuestionAnswering(ErniePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Task type embedding is a special embedding to represent the characteristic of different tasks, such as + word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We + assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, + config.task_type_vocab_size-1] """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/esm/modeling_esm.py b/src/transformers/models/esm/modeling_esm.py index 6f90d8d052..28c6d249c7 100755 --- a/src/transformers/models/esm/modeling_esm.py +++ b/src/transformers/models/esm/modeling_esm.py @@ -22,7 +22,6 @@ import torch.utils.checkpoint from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss -from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -31,15 +30,12 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import logging +from ...utils import auto_docstring, logging from .configuration_esm import EsmConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/esm2_t6_8M_UR50D" -_CONFIG_FOR_DOC = "EsmConfig" - def rotate_half(x): x1, x2 = x.chunk(2, dim=-1) @@ -670,12 +666,8 @@ class EsmPooler(nn.Module): return pooled_output +@auto_docstring class EsmPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = EsmConfig base_model_prefix = "esm" supports_gradient_checkpointing = True @@ -701,68 +693,7 @@ class EsmPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -ESM_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`EsmConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ESM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ESM Model transformer outputting raw hidden-states without any specific head on top.", - ESM_START_DOCSTRING, -) +@auto_docstring class EsmModel(EsmPreTrainedModel): """ @@ -777,6 +708,10 @@ class EsmModel(EsmPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -806,12 +741,7 @@ class EsmModel(EsmPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -828,24 +758,22 @@ class EsmModel(EsmPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`): + Indices of input sequence tokens in the vocabulary. - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). + [What are input IDs?](../glossary#input-ids) + position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length), hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -945,7 +873,7 @@ class EsmModel(EsmPreTrainedModel): return self.contact_head(tokens, attns) -@add_start_docstrings("""ESM Model with a `language modeling` head on top.""", ESM_START_DOCSTRING) +@auto_docstring class EsmForMaskedLM(EsmPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight"] @@ -969,13 +897,7 @@ class EsmForMaskedLM(EsmPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -995,8 +917,6 @@ class EsmForMaskedLM(EsmPreTrainedModel): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1058,12 +978,11 @@ class EsmLMHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ESM_START_DOCSTRING, + """ ) class EsmForSequenceClassification(EsmPreTrainedModel): def __init__(self, config): @@ -1076,12 +995,7 @@ class EsmForSequenceClassification(EsmPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1152,13 +1066,7 @@ class EsmForSequenceClassification(EsmPreTrainedModel): ) -@add_start_docstrings( - """ - ESM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - ESM_START_DOCSTRING, -) +@auto_docstring class EsmForTokenClassification(EsmPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1170,12 +1078,7 @@ class EsmForTokenClassification(EsmPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(ESM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/esm/modeling_esmfold.py b/src/transformers/models/esm/modeling_esmfold.py index 645c9d16a5..c47f87b408 100644 --- a/src/transformers/models/esm/modeling_esmfold.py +++ b/src/transformers/models/esm/modeling_esmfold.py @@ -27,14 +27,11 @@ from ...integrations.deepspeed import is_deepspeed_available from ...modeling_outputs import ModelOutput from ...utils import ( ContextManagers, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_scipy_available, logging, - replace_return_docstrings, ) -from .configuration_esm import EsmConfig -from .modeling_esm import ESM_START_DOCSTRING, EsmModel, EsmPreTrainedModel +from .modeling_esm import EsmModel, EsmPreTrainedModel from .openfold_utils import ( OFProtein, Rigid, @@ -52,8 +49,6 @@ from .openfold_utils import ( logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/esmfold_v1" -_CONFIG_FOR_DOC = "EsmConfig" @dataclass @@ -137,39 +132,6 @@ class EsmForProteinFoldingOutput(ModelOutput): max_predicted_aligned_error: Optional[torch.FloatTensor] = None -ESMFOLD_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - masking_pattern (`torch.LongTensor` of shape `({0})`, *optional*): - Locations of tokens to mask during training as a form of regularization. Mask values selected in `[0, 1]`. - num_recycles (`int`, *optional*, defaults to `None`): - Number of times to recycle the input sequence. If `None`, defaults to `config.num_recycles`. "Recycling" - consists of passing the output of the folding trunk back in as input to the trunk. During training, the - number of recycles should vary with each batch, to ensure that the model learns to output valid predictions - after each recycle. During inference, num_recycles should be set to the highest value that the model was - trained with for maximum accuracy. Accordingly, when this value is set to `None`, config.max_recycles is - used. -""" - - def is_fp16_enabled(): # Autocast world fp16_enabled = torch.get_autocast_gpu_dtype() == torch.float16 @@ -2008,14 +1970,13 @@ class EsmFoldingTrunk(nn.Module): # the outputs for downstream use. -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ESMForProteinFolding is the HuggingFace port of the original ESMFold model. It consists of an ESM-2 "stem" followed by a protein folding "head", although unlike most other output heads, this "head" is similar in size and runtime to the rest of the model combined! It outputs a dictionary containing predicted structural information about the input protein(s). - """, - ESM_START_DOCSTRING, + """ ) class EsmForProteinFolding(EsmPreTrainedModel): _no_split_modules = ["EsmFoldStructureModule", "EsmFoldTriangularSelfAttentionBlock"] @@ -2081,8 +2042,7 @@ class EsmForProteinFolding(EsmPreTrainedModel): esm_reorder = [vocab_list.index("")] + [vocab_list.index(v) for v in residue_constants.restypes_with_x] return torch.tensor(esm_reorder) - @add_start_docstrings_to_model_forward(ESMFOLD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=EsmForProteinFoldingOutput, config_class=EsmConfig) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -2092,7 +2052,15 @@ class EsmForProteinFolding(EsmPreTrainedModel): num_recycles: Optional[int] = None, ) -> EsmForProteinFoldingOutput: r""" - Returns: + masking_pattern (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Locations of tokens to mask during training as a form of regularization. Mask values selected in `[0, 1]`. + num_recycles (`int`, *optional*, defaults to `None`): + Number of times to recycle the input sequence. If `None`, defaults to `config.num_recycles`. "Recycling" + consists of passing the output of the folding trunk back in as input to the trunk. During training, the + number of recycles should vary with each batch, to ensure that the model learns to output valid predictions + after each recycle. During inference, num_recycles should be set to the highest value that the model was + trained with for maximum accuracy. Accordingly, when this value is set to `None`, config.max_recycles is + used. Example: diff --git a/src/transformers/models/falcon/modeling_falcon.py b/src/transformers/models/falcon/modeling_falcon.py index b3f5b12f62..d2d80af9fe 100644 --- a/src/transformers/models/falcon/modeling_falcon.py +++ b/src/transformers/models/falcon/modeling_falcon.py @@ -40,9 +40,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) from .configuration_falcon import FalconConfig @@ -57,10 +55,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Rocketknight1/falcon-rw-1b" -_CONFIG_FOR_DOC = "FalconConfig" - - # NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during training, this means that there's one additional quantization to bfloat16 between the operations. # In order not to degrade the quality of our HF-port, we keep these characteristics in the final model. class FalconLinear(nn.Linear): @@ -655,101 +649,8 @@ class FalconDecoderLayer(nn.Module): return outputs # hidden_states, past_kv, attentions -FALCON_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FalconConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FALCON_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` - (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - +@auto_docstring class FalconPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FalconConfig base_model_prefix = "transformer" supports_gradient_checkpointing = True @@ -791,10 +692,7 @@ class FalconPreTrainedModel(PreTrainedModel): return config -@add_start_docstrings( - "The bare Falcon Model transformer outputting raw hidden-states without any specific head on top.", - FALCON_START_DOCSTRING, -) +@auto_docstring class FalconModel(FalconPreTrainedModel): def __init__(self, config: FalconConfig): super().__init__(config) @@ -827,12 +725,7 @@ class FalconModel(FalconPreTrainedModel): def set_input_embeddings(self, new_embeddings: torch.Tensor): self.word_embeddings = new_embeddings - @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -847,6 +740,19 @@ class FalconModel(FalconPreTrainedModel): return_dict: Optional[bool] = None, cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1126,9 +1032,10 @@ class FalconModel(FalconPreTrainedModel): return causal_mask -@add_start_docstrings( - "The Falcon Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).", - FALCON_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Falcon Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + """ ) class FalconForCausalLM(FalconPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1147,12 +1054,7 @@ class FalconForCausalLM(FalconPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings: torch.Tensor): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1171,17 +1073,21 @@ class FalconForCausalLM(FalconPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1250,8 +1156,8 @@ class FalconForCausalLM(FalconPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Falcon Model transformer with a sequence classification head on top (linear layer). [`FalconForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1262,8 +1168,7 @@ class FalconForCausalLM(FalconPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - FALCON_START_DOCSTRING, + """ ) class FalconForSequenceClassification(FalconPreTrainedModel): def __init__(self, config: FalconConfig): @@ -1275,12 +1180,7 @@ class FalconForSequenceClassification(FalconPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1295,6 +1195,17 @@ class FalconForSequenceClassification(FalconPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1376,13 +1287,7 @@ class FalconForSequenceClassification(FalconPreTrainedModel): ) -@add_start_docstrings( - """ - Falcon Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - FALCON_START_DOCSTRING, -) +@auto_docstring class FalconForTokenClassification(FalconPreTrainedModel): def __init__(self, config: FalconConfig): super().__init__(config) @@ -1401,12 +1306,7 @@ class FalconForTokenClassification(FalconPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1421,6 +1321,17 @@ class FalconForTokenClassification(FalconPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1465,13 +1376,7 @@ class FalconForTokenClassification(FalconPreTrainedModel): ) -@add_start_docstrings( - """ - The Falcon Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - FALCON_START_DOCSTRING, -) +@auto_docstring class FalconForQuestionAnswering(FalconPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1481,7 +1386,7 @@ class FalconForQuestionAnswering(FalconPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FALCON_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1495,14 +1400,17 @@ class FalconForQuestionAnswering(FalconPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py b/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py index bf8695d2a9..7b588d8fa4 100644 --- a/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py +++ b/src/transformers/models/falcon_mamba/modeling_falcon_mamba.py @@ -27,13 +27,7 @@ from ...activations import ACT2FN from ...cache_utils import MambaCache from ...generation import GenerationMixin from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import ModelOutput, auto_docstring, logging from ...utils.import_utils import is_causal_conv1d_available, is_mamba_ssm_available, is_mambapy_available from .configuration_falcon_mamba import FalconMambaConfig @@ -62,9 +56,6 @@ is_fast_path_available = all( (selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn) ) -_CHECKPOINT_FOR_DOC = "tiiuae/falcon-mamba-7b" -_CONFIG_FOR_DOC = "FalconMambaConfig" - def rms_forward(hidden_states, variance_epsilon=1e-6): """ @@ -442,13 +433,9 @@ class FalconMambaBlock(nn.Module): return hidden_states +@auto_docstring # Copied from transformers.models.mamba.modeling_mamba.MambaPreTrainedModel with Mamba->FalconMamba class FalconMambaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FalconMambaConfig base_model_prefix = "backbone" _no_split_modules = ["FalconMambaBlock", "FalconMambaMixer"] @@ -558,55 +545,7 @@ class FalconMambaCausalLMOutput(ModelOutput): hidden_states: Optional[Tuple[torch.FloatTensor]] = None -FALCONMAMBA_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FalconMambaConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FALCONMAMBA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - Indices of input sequence tokens in the vocabulary. - - If `cache_params.seqlen_offset>0`, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - cache_params (`MambaCache`, *optional*): - If passed along, the model uses the previous state in all the blocks (which will give the output for the - `input_ids` provided as if the model add `state_input_ids + input_ids` as context). - use_cache (`bool`, *optional*): - If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare FALCONMAMBA Model transformer outputting raw hidden-states without any specific head on top.", - FALCONMAMBA_START_DOCSTRING, -) +@auto_docstring class FalconMambaModel(FalconMambaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -627,12 +566,7 @@ class FalconMambaModel(FalconMambaPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.embeddings = new_embeddings - @add_start_docstrings_to_model_forward(FALCONMAMBA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=FalconMambaOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -644,6 +578,13 @@ class FalconMambaModel(FalconMambaPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, ) -> Union[Tuple, FalconMambaOutput]: + r""" + cache_params (`MambaCache`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). + use_cache (`bool`, *optional*): + If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. + """ output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -709,12 +650,11 @@ class FalconMambaModel(FalconMambaPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The FALCONMAMBA Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - FALCONMAMBA_START_DOCSTRING, + """ ) # Copied from transformers.models.mamba.modeling_mamba.MambaForCausalLM with MAMBA->FALCONMAMBA,Mamba->FalconMamba,mamba->falcon_mamba,FalconMambaCache->MambaCache class FalconMambaForCausalLM(FalconMambaPreTrainedModel, GenerationMixin): @@ -806,12 +746,7 @@ class FalconMambaForCausalLM(FalconMambaPreTrainedModel, GenerationMixin): ) return model_inputs - @add_start_docstrings_to_model_forward(FALCONMAMBA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=FalconMambaCausalLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -826,10 +761,15 @@ class FalconMambaForCausalLM(FalconMambaPreTrainedModel, GenerationMixin): **kwargs, # for now we need this for generation ) -> Union[Tuple, FalconMambaCausalLMOutput]: r""" + cache_params (`MambaCache`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + use_cache (`bool`, *optional*): + If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/fastspeech2_conformer/modeling_fastspeech2_conformer.py b/src/transformers/models/fastspeech2_conformer/modeling_fastspeech2_conformer.py index a9faf1d03e..ed78a8a7e5 100644 --- a/src/transformers/models/fastspeech2_conformer/modeling_fastspeech2_conformer.py +++ b/src/transformers/models/fastspeech2_conformer/modeling_fastspeech2_conformer.py @@ -23,7 +23,7 @@ from torch import nn from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ModelOutput, add_start_docstrings, logging, replace_return_docstrings +from ...utils import ModelOutput, auto_docstring, logging from .configuration_fastspeech2_conformer import ( FastSpeech2ConformerConfig, FastSpeech2ConformerHifiGanConfig, @@ -136,58 +136,6 @@ class FastSpeech2ConformerWithHifiGanOutput(FastSpeech2ConformerModelOutput): waveform: Optional[torch.FloatTensor] = None -_CONFIG_FOR_DOC = "FastSpeech2ConformerConfig" - -FASTSPEECH2_CONFORMER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FastSpeech2ConformerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -HIFIGAN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FastSpeech2ConformerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FASTSPEECH2_CONFORMER_WITH_HIFIGAN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FastSpeech2ConformerWithHifiGanConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - def length_regulator(encoded_embeddings, duration_labels, speaking_speed=1.0): """ Length regulator for feed-forward Transformer. @@ -1058,12 +1006,8 @@ class FastSpeech2ConformerLoss(nn.Module): return l1_loss + duration_loss + pitch_loss + energy_loss +@auto_docstring class FastSpeech2ConformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FastSpeech2ConformerConfig base_model_prefix = "fastspeech2_conformer" @@ -1092,9 +1036,10 @@ class FastSpeech2ConformerPreTrainedModel(PreTrainedModel): module.gradient_checkpointing = value -@add_start_docstrings( - """FastSpeech2Conformer Model.""", - FASTSPEECH2_CONFORMER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + FastSpeech2Conformer Model. + """ ) class FastSpeech2ConformerModel(FastSpeech2ConformerPreTrainedModel): """ @@ -1173,7 +1118,7 @@ class FastSpeech2ConformerModel(FastSpeech2ConformerPreTrainedModel): self.post_init() - @replace_return_docstrings(output_type=FastSpeech2ConformerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor, @@ -1189,37 +1134,23 @@ class FastSpeech2ConformerModel(FastSpeech2ConformerPreTrainedModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> Union[Tuple, FastSpeech2ConformerModelOutput]: - """ - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Input sequence of text vectors. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to `None`): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in - `[0, 1]`: 0 for tokens that are **masked**, 1 for tokens that are **not masked**. - spectrogram_labels (`torch.FloatTensor` of shape `(batch_size, max_spectrogram_length, num_mel_bins)`, *optional*, defaults to `None`): - Batch of padded target features. - duration_labels (`torch.LongTensor` of shape `(batch_size, sequence_length + 1)`, *optional*, defaults to `None`): - Batch of padded durations. - pitch_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): - Batch of padded token-averaged pitch. - energy_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): - Batch of padded token-averaged energy. - speaker_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): - Speaker ids used to condition features of speech output by the model. - lang_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): - Language ids used to condition features of speech output by the model. - speaker_embedding (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`, *optional*, defaults to `None`): - Embedding containing conditioning signals for the features of the speech. - return_dict (`bool`, *optional*, defaults to `None`): - Whether or not to return a [`FastSpeech2ConformerModelOutput`] instead of a plain tuple. - output_attentions (`bool`, *optional*, defaults to `None`): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*, defaults to `None`): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - - Returns: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Input sequence of text vectors. + spectrogram_labels (`torch.FloatTensor` of shape `(batch_size, max_spectrogram_length, num_mel_bins)`, *optional*, defaults to `None`): + Batch of padded target features. + duration_labels (`torch.LongTensor` of shape `(batch_size, sequence_length + 1)`, *optional*, defaults to `None`): + Batch of padded durations. + pitch_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): + Batch of padded token-averaged pitch. + energy_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): + Batch of padded token-averaged energy. + speaker_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): + Speaker ids used to condition features of speech output by the model. + lang_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): + Language ids used to condition features of speech output by the model. + speaker_embedding (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`, *optional*, defaults to `None`): + Embedding containing conditioning signals for the features of the speech. Example: @@ -1442,9 +1373,10 @@ class HifiGanResidualBlock(nn.Module): return hidden_states -@add_start_docstrings( - """HiFi-GAN vocoder.""", - HIFIGAN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + HiFi-GAN vocoder. + """ ) # Copied from transformers.models.speecht5.modeling_speecht5.SpeechT5HifiGan with SpeechT5->FastSpeech2Conformer class FastSpeech2ConformerHifiGan(PreTrainedModel): @@ -1516,16 +1448,18 @@ class FastSpeech2ConformerHifiGan(PreTrainedModel): layer.remove_weight_norm() nn.utils.remove_weight_norm(self.conv_post) - def forward(self, spectrogram: torch.FloatTensor) -> torch.FloatTensor: - r""" + @auto_docstring( + custom_intro=""" Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech waveform. - - Args: - spectrogram (`torch.FloatTensor`): - Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, - config.model_in_dim)`, or un-batched and of shape `(sequence_length, config.model_in_dim)`. + """ + ) + def forward(self, spectrogram: torch.FloatTensor) -> torch.FloatTensor: + r""" + spectrogram (`torch.FloatTensor`): + Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, + config.model_in_dim)`, or un-batched and of shape `(sequence_length, config.model_in_dim)`. Returns: `torch.FloatTensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of @@ -1564,9 +1498,10 @@ class FastSpeech2ConformerHifiGan(PreTrainedModel): return waveform -@add_start_docstrings( - "The FastSpeech2ConformerModel with a FastSpeech2ConformerHifiGan vocoder head that performs text-to-speech (waveform).", - FASTSPEECH2_CONFORMER_WITH_HIFIGAN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The FastSpeech2ConformerModel with a FastSpeech2ConformerHifiGan vocoder head that performs text-to-speech (waveform). + """ ) class FastSpeech2ConformerWithHifiGan(PreTrainedModel): config_class = FastSpeech2ConformerWithHifiGanConfig @@ -1579,9 +1514,7 @@ class FastSpeech2ConformerWithHifiGan(PreTrainedModel): self.config = config - @replace_return_docstrings( - output_type=FastSpeech2ConformerWithHifiGanOutput, config_class=FastSpeech2ConformerWithHifiGanConfig - ) + @auto_docstring def forward( self, input_ids: torch.LongTensor, @@ -1597,37 +1530,23 @@ class FastSpeech2ConformerWithHifiGan(PreTrainedModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> Union[Tuple, FastSpeech2ConformerModelOutput]: - """ - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Input sequence of text vectors. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to `None`): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in - `[0, 1]`: 0 for tokens that are **masked**, 1 for tokens that are **not masked**. - spectrogram_labels (`torch.FloatTensor` of shape `(batch_size, max_spectrogram_length, num_mel_bins)`, *optional*, defaults to `None`): - Batch of padded target features. - duration_labels (`torch.LongTensor` of shape `(batch_size, sequence_length + 1)`, *optional*, defaults to `None`): - Batch of padded durations. - pitch_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): - Batch of padded token-averaged pitch. - energy_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): - Batch of padded token-averaged energy. - speaker_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): - Speaker ids used to condition features of speech output by the model. - lang_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): - Language ids used to condition features of speech output by the model. - speaker_embedding (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`, *optional*, defaults to `None`): - Embedding containing conditioning signals for the features of the speech. - return_dict (`bool`, *optional*, defaults to `None`): - Whether or not to return a [`FastSpeech2ConformerModelOutput`] instead of a plain tuple. - output_attentions (`bool`, *optional*, defaults to `None`): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*, defaults to `None`): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - - Returns: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Input sequence of text vectors. + spectrogram_labels (`torch.FloatTensor` of shape `(batch_size, max_spectrogram_length, num_mel_bins)`, *optional*, defaults to `None`): + Batch of padded target features. + duration_labels (`torch.LongTensor` of shape `(batch_size, sequence_length + 1)`, *optional*, defaults to `None`): + Batch of padded durations. + pitch_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): + Batch of padded token-averaged pitch. + energy_labels (`torch.FloatTensor` of shape `(batch_size, sequence_length + 1, 1)`, *optional*, defaults to `None`): + Batch of padded token-averaged energy. + speaker_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): + Speaker ids used to condition features of speech output by the model. + lang_ids (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*, defaults to `None`): + Language ids used to condition features of speech output by the model. + speaker_embedding (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`, *optional*, defaults to `None`): + Embedding containing conditioning signals for the features of the speech. Example: diff --git a/src/transformers/models/flaubert/modeling_flaubert.py b/src/transformers/models/flaubert/modeling_flaubert.py index 33a312bdee..8eb2fc4b1c 100644 --- a/src/transformers/models/flaubert/modeling_flaubert.py +++ b/src/transformers/models/flaubert/modeling_flaubert.py @@ -36,22 +36,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_flaubert import FlaubertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "flaubert/flaubert_base_cased" -_CONFIG_FOR_DOC = "FlaubertConfig" - # Copied from transformers.models.xlm.modeling_xlm.create_sinusoidal_embeddings def create_sinusoidal_embeddings(n_pos, dim, out): @@ -207,84 +197,10 @@ class TransformerFFN(nn.Module): return x -FLAUBERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FlaubertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FLAUBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Length of each sentence that can be used to avoid performing attention on padding token indices. You can - also use `attention_mask` for the same result (see above), kept here for compatibility. Indices selected in - `[0, ..., input_ids.size(-1)]`: - cache (`Dict[str, torch.FloatTensor]`, *optional*): - Dictionary strings to `torch.FloatTensor` that contains precomputed hidden-states (key and values in the - attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential - decoding. The dictionary object will be modified in-place during the forward pass to add newly computed - hidden-states. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Flaubert Model transformer outputting raw hidden-states without any specific head on top.", - FLAUBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare Flaubert Model transformer outputting raw hidden-states without any specific head on top. + """ ) # Copied from transformers.models.xlm.modeling_xlm.XLMPredLayer with XLM->Flaubert class FlaubertPredLayer(nn.Module): @@ -556,7 +472,7 @@ class FlaubertSQuADHead(nn.Module): self.end_logits = FlaubertPoolerEndLogits(config) self.answer_class = FlaubertPoolerAnswerClass(config) - @replace_return_docstrings(output_type=FlaubertSquadHeadOutput, config_class=FlaubertConfig) + @auto_docstring def forward( self, hidden_states: torch.FloatTensor, @@ -567,25 +483,20 @@ class FlaubertSQuADHead(nn.Module): p_mask: Optional[torch.FloatTensor] = None, return_dict: bool = False, ) -> Union[FlaubertSquadHeadOutput, Tuple[torch.FloatTensor]]: - """ - Args: - hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): - Final hidden states of the model on the sequence tokens. - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Positions of the first token for the labeled span. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Positions of the last token for the labeled span. - cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Position of the CLS token for each sentence in the batch. If `None`, takes the last token. - is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Whether the question has a possible answer in the paragraph or not. - p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): - Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token - should be masked. - return_dict (`bool`, *optional*, defaults to `False`): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + r""" + hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): + Final hidden states of the model on the sequence tokens. + start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Positions of the first token for the labeled span. + end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Positions of the last token for the labeled span. + cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Position of the CLS token for each sentence in the batch. If `None`, takes the last token. + is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Whether the question has a possible answer in the paragraph or not. + p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): + Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token + should be masked. """ start_logits = self.start_logits(hidden_states, p_mask=p_mask) @@ -754,13 +665,9 @@ class FlaubertSequenceSummary(nn.Module): return output +@auto_docstring # Copied from transformers.models.xlm.modeling_xlm.XLMPreTrainedModel with XLM->Flaubert class FlaubertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FlaubertConfig load_tf_weights = None base_model_prefix = "transformer" @@ -799,6 +706,7 @@ class FlaubertPreTrainedModel(PreTrainedModel): ) +@auto_docstring class FlaubertModel(FlaubertPreTrainedModel): def __init__(self, config): # , dico, is_encoder, with_output): super().__init__(config) @@ -890,12 +798,7 @@ class FlaubertModel(FlaubertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.attentions[layer].prune_heads(heads) - @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -911,6 +814,25 @@ class FlaubertModel(FlaubertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: + r""" + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use `attention_mask` for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`: + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary strings to `torch.FloatTensor` that contains precomputed hidden-states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1062,14 +984,12 @@ class FlaubertModel(FlaubertPreTrainedModel): return BaseModelOutput(last_hidden_state=tensor, hidden_states=hidden_states, attentions=attentions) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Flaubert Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - FLAUBERT_START_DOCSTRING, + """ ) -# Copied transformers.models.xlm.modeling_xlm.XLMWithLMHeadModel with XLM_INPUTS->FLAUBERT_INPUTS,XLM->Flaubert class FlaubertWithLMHeadModel(FlaubertPreTrainedModel, GenerationMixin): _tied_weights_keys = ["pred_layer.proj.weight"] @@ -1102,13 +1022,7 @@ class FlaubertWithLMHeadModel(FlaubertPreTrainedModel, GenerationMixin): langs = None return {"input_ids": input_ids, "langs": langs} - @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1126,10 +1040,27 @@ class FlaubertWithLMHeadModel(FlaubertPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, MaskedLMOutput]: r""" + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use `attention_mask` for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`: + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary strings to `torch.FloatTensor` that contains precomputed hidden-states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1162,12 +1093,11 @@ class FlaubertWithLMHeadModel(FlaubertPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Flaubert Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - FLAUBERT_START_DOCSTRING, + """ ) # Copied from transformers.models.xlm.modeling_xlm.XLMForSequenceClassification with XLM_INPUTS->FLAUBERT_INPUTS,XLM->Flaubert class FlaubertForSequenceClassification(FlaubertPreTrainedModel): @@ -1182,12 +1112,7 @@ class FlaubertForSequenceClassification(FlaubertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1205,6 +1130,25 @@ class FlaubertForSequenceClassification(FlaubertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1265,13 +1209,7 @@ class FlaubertForSequenceClassification(FlaubertPreTrainedModel): ) -@add_start_docstrings( - """ - Flaubert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - FLAUBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.xlm.modeling_xlm.XLMForTokenClassification with XLM_INPUTS->FLAUBERT_INPUTS,XLM->Flaubert class FlaubertForTokenClassification(FlaubertPreTrainedModel): def __init__(self, config): @@ -1285,12 +1223,7 @@ class FlaubertForTokenClassification(FlaubertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1308,6 +1241,25 @@ class FlaubertForTokenClassification(FlaubertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1350,12 +1302,11 @@ class FlaubertForTokenClassification(FlaubertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Flaubert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - FLAUBERT_START_DOCSTRING, + """ ) # Copied from transformers.models.xlm.modeling_xlm.XLMForQuestionAnsweringSimple with XLM_INPUTS->FLAUBERT_INPUTS,XLM->Flaubert class FlaubertForQuestionAnsweringSimple(FlaubertPreTrainedModel): @@ -1368,12 +1319,7 @@ class FlaubertForQuestionAnsweringSimple(FlaubertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1392,14 +1338,25 @@ class FlaubertForQuestionAnsweringSimple(FlaubertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1455,13 +1412,6 @@ class FlaubertForQuestionAnsweringSimple(FlaubertPreTrainedModel): ) -@add_start_docstrings( - """ - Flaubert Model with a beam-search span classification head on top for extractive question-answering tasks like - SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - FLAUBERT_START_DOCSTRING, -) @dataclass # Copied from transformer.models.xlm.modeling_xlm.XLMForQuestionAnsweringOutput with XLM->Flaubert class FlaubertForQuestionAnsweringOutput(ModelOutput): @@ -1506,6 +1456,7 @@ class FlaubertForQuestionAnsweringOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@auto_docstring # Copied from transformers.models.xlm.modeling_xlm.XLMForQuestionAnswering with XLM_INPUTS->FLAUBERT_INPUTS,XLM->Flaubert class FlaubertForQuestionAnswering(FlaubertPreTrainedModel): def __init__(self, config): @@ -1517,8 +1468,7 @@ class FlaubertForQuestionAnswering(FlaubertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FLAUBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=FlaubertForQuestionAnsweringOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1540,14 +1490,25 @@ class FlaubertForQuestionAnswering(FlaubertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, FlaubertForQuestionAnsweringOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels whether a question has an answer or no answer (SQuAD 2.0) cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): @@ -1557,8 +1518,6 @@ class FlaubertForQuestionAnswering(FlaubertPreTrainedModel): Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be masked. 0.0 mean token is not masked. - Returns: - Example: ```python @@ -1621,13 +1580,7 @@ class FlaubertForQuestionAnswering(FlaubertPreTrainedModel): ) -@add_start_docstrings( - """ - Flaubert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - FLAUBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.xlm.modeling_xlm.XLMForMultipleChoice with XLM_INPUTS->FLAUBERT_INPUTS,XLM->Flaubert class FlaubertForMultipleChoice(FlaubertPreTrainedModel): def __init__(self, config, *inputs, **kwargs): @@ -1640,14 +1593,7 @@ class FlaubertForMultipleChoice(FlaubertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - FLAUBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1665,6 +1611,49 @@ class FlaubertForMultipleChoice(FlaubertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + langs (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See diff --git a/src/transformers/models/flava/image_processing_flava_fast.py b/src/transformers/models/flava/image_processing_flava_fast.py index 28d1715820..09e0606f78 100644 --- a/src/transformers/models/flava/image_processing_flava_fast.py +++ b/src/transformers/models/flava/image_processing_flava_fast.py @@ -20,8 +20,6 @@ from functools import lru_cache from typing import Any, Dict, Iterable, Optional, Tuple, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -32,7 +30,7 @@ from ...image_utils import ImageInput, PILImageResampling, SizeDict from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -135,33 +133,8 @@ class FlavaMaskingGenerator: class FlavaFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): - # Mask related params - return_image_mask: Optional[bool] - input_size_patches: Optional[int] - total_mask_patches: Optional[int] - mask_group_min_patches: Optional[int] - mask_group_max_patches: Optional[int] - mask_group_min_aspect_ratio: Optional[float] - mask_group_max_aspect_ratio: Optional[float] - # Codebook related params - return_codebook_pixels: Optional[bool] - codebook_do_resize: Optional[bool] - codebook_size: Optional[bool] - codebook_resample: Optional[int] - codebook_do_center_crop: Optional[bool] - codebook_crop_size: Optional[int] - codebook_do_rescale: Optional[bool] - codebook_rescale_factor: Optional[Union[int, float]] - codebook_do_map_pixels: Optional[bool] - codebook_do_normalize: Optional[bool] - codebook_image_mean: Optional[Union[float, Iterable[float]]] - codebook_image_std: Optional[Union[float, Iterable[float]]] - - -@add_start_docstrings( - "Constructs a fast Flava image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, """ + Args: return_image_mask (`bool`, *optional*, defaults to `False`): Whether to return the image mask. Can be overridden by the `return_image_mask` parameter in `preprocess`. input_size_patches (`int`, *optional*, defaults to 14): @@ -182,6 +155,8 @@ class FlavaFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): mask_group_max_aspect_ratio (`float`, *optional*): Maximum aspect ratio of the mask window. Can be overridden by the `mask_group_max_aspect_ratio` parameter in `preprocess`. + return_codebook_pixels (`bool`, *optional*, defaults to `False`): + Whether to return the codebook pixel values. codebook_do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the input for codebook to a certain. Can be overridden by the `codebook_do_resize` parameter in `preprocess`. `codebook_size`. @@ -216,8 +191,32 @@ class FlavaFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): codebook_image_std (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0.5, 0.5, 0.5]`): The sequence of standard deviations for each channel, to be used when normalizing images for codebook. Can be overridden by the `codebook_image_std` parameter in `preprocess`. - """, -) + """ + + # Mask related params + return_image_mask: Optional[bool] + input_size_patches: Optional[int] + total_mask_patches: Optional[int] + mask_group_min_patches: Optional[int] + mask_group_max_patches: Optional[int] + mask_group_min_aspect_ratio: Optional[float] + mask_group_max_aspect_ratio: Optional[float] + # Codebook related params + return_codebook_pixels: Optional[bool] + codebook_do_resize: Optional[bool] + codebook_size: Optional[bool] + codebook_resample: Optional[int] + codebook_do_center_crop: Optional[bool] + codebook_crop_size: Optional[int] + codebook_do_rescale: Optional[bool] + codebook_rescale_factor: Optional[Union[int, float]] + codebook_do_map_pixels: Optional[bool] + codebook_do_normalize: Optional[bool] + codebook_image_mean: Optional[Union[float, Iterable[float]]] + codebook_image_std: Optional[Union[float, Iterable[float]]] + + +@auto_docstring class FlavaImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = FLAVA_IMAGE_MEAN @@ -256,65 +255,7 @@ class FlavaImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[FlavaFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - return_image_mask (`bool`, *optional*, defaults to `False`): - Whether to return the image mask. Can be overridden by the `return_image_mask` parameter in `preprocess`. - input_size_patches (`int`, *optional*, defaults to 14): - Number of patches in the image in height and width direction. 14x14 = 196 total patches. Can be overridden - by the `input_size_patches` parameter in `preprocess`. - total_mask_patches (`int`, *optional*, defaults to 75): - Total number of patches that should be masked. Can be overridden by the `total_mask_patches` parameter in - `preprocess`. - mask_group_min_patches (`int`, *optional*, defaults to 16): - Minimum number of patches that should be masked. Can be overridden by the `mask_group_min_patches` - parameter in `preprocess`. - mask_group_max_patches (`int`, *optional*): - Maximum number of patches that should be masked. Can be overridden by the `mask_group_max_patches` - parameter in `preprocess`. - mask_group_min_aspect_ratio (`float`, *optional*, defaults to 0.3): - Minimum aspect ratio of the mask window. Can be overridden by the `mask_group_min_aspect_ratio` parameter - in `preprocess`. - mask_group_max_aspect_ratio (`float`, *optional*): - Maximum aspect ratio of the mask window. Can be overridden by the `mask_group_max_aspect_ratio` parameter - in `preprocess`. - codebook_do_resize (`bool`, *optional*, defaults to `True`): - Whether to resize the input for codebook to a certain. Can be overridden by the `codebook_do_resize` - parameter in `preprocess`. `codebook_size`. - codebook_size (`Dict[str, int]`, *optional*, defaults to `{"height": 224, "width": 224}`): - Resize the input for codebook to the given size. Can be overridden by the `codebook_size` parameter in - `preprocess`. - codebook_resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.LANCZOS`): - Resampling filter to use if resizing the codebook image. Can be overridden by the `codebook_resample` - parameter in `preprocess`. - codebook_do_center_crop (`bool`, *optional*, defaults to `True`): - Whether to crop the input for codebook at the center. If the input size is smaller than - `codebook_crop_size` along any edge, the image is padded with 0's and then center cropped. Can be - overridden by the `codebook_do_center_crop` parameter in `preprocess`. - codebook_crop_size (`Dict[str, int]`, *optional*, defaults to `{"height": 224, "width": 224}`): - Desired output size for codebook input when applying center-cropping. Can be overridden by the - `codebook_crop_size` parameter in `preprocess`. - codebook_do_rescale (`bool`, *optional*, defaults to `True`): - Whether to rescale the input for codebook by the specified scale `codebook_rescale_factor`. Can be - overridden by the `codebook_do_rescale` parameter in `preprocess`. - codebook_rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): - Defines the scale factor to use if rescaling the codebook image. Can be overridden by the - `codebook_rescale_factor` parameter in `preprocess`. - codebook_do_map_pixels (`bool`, *optional*, defaults to `True`): - Whether to map the pixel values of the codebook input to (1 - 2e)x + e. Can be overridden by the - `codebook_do_map_pixels` parameter in `preprocess`. - codebook_do_normalize (`bool`, *optional*, defaults to `True`): - Whether or not to normalize the input for codebook with `codebook_image_mean` and `codebook_image_std`. Can - be overridden by the `codebook_do_normalize` parameter in `preprocess`. - codebook_image_mean (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0, 0, 0]`): - The sequence of means for each channel, to be used when normalizing images for codebook. Can be overridden - by the `codebook_image_mean` parameter in `preprocess`. - codebook_image_std (`Optional[Union[float, Iterable[float]]]`, *optional*, defaults to `[0.5, 0.5, 0.5]`): - The sequence of standard deviations for each channel, to be used when normalizing images for codebook. Can - be overridden by the `codebook_image_std` parameter in `preprocess`. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[DefaultFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/flava/modeling_flava.py b/src/transformers/models/flava/modeling_flava.py index 8526771d79..f116d33908 100644 --- a/src/transformers/models/flava/modeling_flava.py +++ b/src/transformers/models/flava/modeling_flava.py @@ -27,15 +27,7 @@ from torch import nn from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_flava import ( FlavaConfig, FlavaImageCodebookConfig, @@ -47,15 +39,7 @@ from .configuration_flava import ( logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/flava-full" - -# Codebook docstring _CHECKPOINT_FOR_CODEBOOK_DOC = "facebook/flava-image-codebook" -_CONFIG_CLASS_FOR_IMAGE_MODEL_DOC = "FlavaImageConfig" -_CONFIG_CLASS_FOR_TEXT_MODEL_DOC = "FlavaTextConfig" -_CONFIG_CLASS_FOR_MULTIMODAL_MODEL_DOC = "FlavaMultimodalConfig" -_EXPECTED_IMAGE_OUTPUT_SHAPE = [1, 197, 768] - LOGIT_SCALE_CLAMP_MIN = 0 LOGIT_SCALE_CLAMP_MAX = 4.6052 @@ -705,156 +689,8 @@ class FlavaPooler(nn.Module): return pooled_output -FLAVA_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`{config}`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FLAVA_INPUTS_DOCSTRING_COMMON = r""" - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -FLAVA_IMAGE_INPUTS_DOCSTRING_BASE = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`FlavaImageProcessor.__call__`] for details. - - bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - - interpolate_pos_encoding (`bool`, *optional*): - Whether to interpolate the pre-trained position encodings. -""" - -FLAVA_IMAGE_INPUTS_DOCSTRING = FLAVA_IMAGE_INPUTS_DOCSTRING_BASE + FLAVA_INPUTS_DOCSTRING_COMMON - -FLAVA_TEXT_INPUTS_DOCSTRING_BASE = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - [What are token type IDs?](../glossary#token-type-ids) -""" - -FLAVA_TEXT_INPUTS_DOCSTRING = FLAVA_TEXT_INPUTS_DOCSTRING_BASE + FLAVA_INPUTS_DOCSTRING_COMMON - -FLAVA_MULTIMODAL_INPUTS_DOCSTRING = ( - r""" - Args: - hidden_states (`torch.FloatTensor` of shape `(batch_size, image_num_patches + text_seq_len, hidden_size)`): - The concatenated hidden states of unimodal encoders. -""" - + FLAVA_INPUTS_DOCSTRING_COMMON -) - -FLAVA_MODEL_INPUTS_DOCSTRING_BASE = r""" - Args: - skip_multimodal_encoder (*bool*, *optional*): - Skip any calculations for multimodal encoder. Useful if multimodal encoding is not going to be used. -""" - -FLAVA_MODEL_INPUTS_DOCSTRING = ( - FLAVA_IMAGE_INPUTS_DOCSTRING_BASE - + FLAVA_TEXT_INPUTS_DOCSTRING_BASE - + FLAVA_INPUTS_DOCSTRING_COMMON - + FLAVA_MODEL_INPUTS_DOCSTRING_BASE -) - - -FLAVA_PRETRAINING_INPUTS_DOCSTRING = ( - r""" - Args: - input_ids_masked (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. These ones are the masked version of the original task - to be used with MLM. Indices can be obtained using [`AutoTokenizer`] along with - [`DataCollatorForMaskedLanguageModeling`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - -""" - + FLAVA_TEXT_INPUTS_DOCSTRING_BASE - + FLAVA_IMAGE_INPUTS_DOCSTRING_BASE - + r""" - image_attention_mask (`torch.FloatTensor` of shape `({1})`, *optional*): - Mask to avoid performing attention on padding token indices specifically for images. Mask values selected - in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - skip_unmasked_multimodal_encoder (*bool*, *optional*): - Skip any calculations for multimodal encoder for unmasked inputs. FLAVA pretraining doesn't need unmasked - multimodal embeddings or outputs as of now. - - mlm_labels (`torch.LongTensor` of shape `(batch_size, text_seq_len)`, *optional*): - Labels for computing the left-to-right language and multimodal masked modeling loss (next word prediction). - Indices should be in `[-100, 0, ..., text_config.vocab_size - 1]` (see `input_ids` docstring). Tokens with - indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, - ..., text_config.vocab_size - 1]`. - - mim_labels (`torch.LongTensor` of shape `(batch_size, image_num_patches)`, *optional*): - Labels for computing the image and multimodal masked modeling loss. Indices should be in `[-100, 0, ..., - image_config.vocab_size - 1]`. Tokens with indices set to `-100` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., image_config.vocab_size - 1]`. If not passed, they are - generated automatically using the image codebook assigned to the model. By default, it uses - [`FlavaImageCodebook`]. See [`FlavaImageCodebook`] to understand how to generate mim_labels. - - itm_labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*): - Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match. - The pairs with 0 will be skipped for calculation of MMM and global contrastive losses as well. - - return_loss (`bool`, *optional*, default to None): - Whether to return calculated loss or not. -""" - + FLAVA_INPUTS_DOCSTRING_COMMON -) - -FLAVA_PRETRAINING_START_DOCSTRING_EXTRA = r""" - Parameters: - image_codebook ([`nn.Module`]): If passed, the image codebook will be set to this. Otherwise. it will - be initialized using the image_codebook_config defined in the config first as the first parameter. -""" - - +@auto_docstring class FlavaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FlavaConfig base_model_prefix = "flava" supports_gradient_checkpointing = True @@ -888,10 +724,7 @@ class FlavaPreTrainedModel(PreTrainedModel): module.logit_scale.data.fill_(self.config.logit_scale_init_value) -@add_start_docstrings( - "The bare FLAVA Image Model transformer outputting raw hidden-states without any specific head on top.", - FLAVA_START_DOCSTRING.format(config="FlavaImageConfig"), -) +@auto_docstring class FlavaImageModel(FlavaPreTrainedModel): config_class = FlavaImageConfig # This override allows us to load FlavaImageModel from FlavaModel/FlavaForPreTraining checkpoints. @@ -899,6 +732,10 @@ class FlavaImageModel(FlavaPreTrainedModel): main_input_name = "pixel_values" def __init__(self, config: FlavaImageConfig, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -925,14 +762,7 @@ class FlavaImageModel(FlavaPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(FLAVA_IMAGE_INPUTS_DOCSTRING.format("batch_size, image_num_patches")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_CLASS_FOR_IMAGE_MODEL_DOC, - modality="vision", - expected_output=_EXPECTED_IMAGE_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -944,6 +774,10 @@ class FlavaImageModel(FlavaPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[tuple, BaseModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -987,16 +821,17 @@ class FlavaImageModel(FlavaPreTrainedModel): ) -@add_start_docstrings( - "The bare FLAVA Text Model transformer outputting raw hidden-states without any specific head on top.", - FLAVA_START_DOCSTRING.format(config="FlavaTextConfig"), -) +@auto_docstring class FlavaTextModel(FlavaPreTrainedModel): config_class = FlavaTextConfig # This override allows us to load FlavaTextModel from FlavaModel/FlavaForPreTraining checkpoints. base_model_prefix = "flava.text_model" def __init__(self, config: FlavaTextConfig, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1022,12 +857,7 @@ class FlavaTextModel(FlavaPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(FLAVA_TEXT_INPUTS_DOCSTRING.format("batch_size, text_seq_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_CLASS_FOR_TEXT_MODEL_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1039,6 +869,18 @@ class FlavaTextModel(FlavaPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[tuple, BaseModelOutputWithPooling]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + [What are token type IDs?](../glossary#token-type-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1092,10 +934,7 @@ class FlavaTextModel(FlavaPreTrainedModel): ) -@add_start_docstrings( - "The bare FLAVA Multimodal Model transformer outputting raw hidden-states without any specific head on top.", - FLAVA_START_DOCSTRING.format(config="FlavaMultimodalConfig"), -) +@auto_docstring class FlavaMultimodalModel(FlavaPreTrainedModel): config_class = FlavaMultimodalConfig # This override allows us to load FlavaMultimodalModel from FlavaModel/FlavaForPreTraining checkpoints. @@ -1103,6 +942,10 @@ class FlavaMultimodalModel(FlavaPreTrainedModel): main_input_name = "hidden_states" def __init__(self, config: FlavaMultimodalConfig, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config self.use_cls_token = self.config.use_cls_token @@ -1124,14 +967,7 @@ class FlavaMultimodalModel(FlavaPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward( - FLAVA_MULTIMODAL_INPUTS_DOCSTRING.format("batch_size, image_num_patches + text_seq_len") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_CLASS_FOR_MULTIMODAL_MODEL_DOC, - ) + @auto_docstring def forward( self, hidden_states: torch.Tensor, @@ -1141,6 +977,10 @@ class FlavaMultimodalModel(FlavaPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[tuple, BaseModelOutputWithPooling]: + r""" + hidden_states (`torch.FloatTensor` of shape `(batch_size, image_num_patches + text_seq_len, hidden_size)`): + The concatenated hidden states of unimodal encoders. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1190,10 +1030,7 @@ class FlavaMultimodalModel(FlavaPreTrainedModel): ) -@add_start_docstrings( - "The bare FLAVA Model transformer outputting raw hidden-states without any specific head on top.", - FLAVA_START_DOCSTRING.format(config="FlavaConfig"), -) +@auto_docstring class FlavaModel(FlavaPreTrainedModel): config_class = FlavaConfig @@ -1240,7 +1077,7 @@ class FlavaModel(FlavaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FLAVA_TEXT_INPUTS_DOCSTRING.format("batch_size, text_seq_length")) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1252,6 +1089,17 @@ class FlavaModel(FlavaPreTrainedModel): return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, text_seq_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + [What are token type IDs?](../glossary#token-type-ids) + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`FlavaTextModel`]. @@ -1268,7 +1116,8 @@ class FlavaModel(FlavaPreTrainedModel): ... text=["a photo of a cat", "a photo of a dog"], max_length=77, padding="max_length", return_tensors="pt" ... ) >>> text_features = model.get_text_features(**inputs) - ```""".format(_CHECKPOINT_FOR_DOC) + ``` + """ text_outputs = self.text_model( input_ids=input_ids, attention_mask=attention_mask, @@ -1284,7 +1133,7 @@ class FlavaModel(FlavaPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(FLAVA_IMAGE_INPUTS_DOCSTRING.format("batch_size, image_num_patches")) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.Tensor] = None, @@ -1297,6 +1146,9 @@ class FlavaModel(FlavaPreTrainedModel): return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`FlavaImageModel`]. @@ -1317,7 +1169,8 @@ class FlavaModel(FlavaPreTrainedModel): >>> inputs = processor(images=image, return_tensors="pt") >>> image_features = model.get_image_features(**inputs) - ```""".format(_CHECKPOINT_FOR_DOC) + ``` + """ image_outputs = self.image_model( pixel_values=pixel_values, bool_masked_pos=bool_masked_pos, @@ -1334,10 +1187,7 @@ class FlavaModel(FlavaPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward( - FLAVA_MODEL_INPUTS_DOCSTRING.format("batch_size, image_num_patches + text_seq_len") - ) - @replace_return_docstrings(output_type=FlavaModelOutput, config_class=FlavaConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1353,7 +1203,24 @@ class FlavaModel(FlavaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, FlavaOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, image_num_patches + text_seq_len)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, image_num_patches + text_seq_len)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + [What are token type IDs?](../glossary#token-type-ids) + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + skip_multimodal_encoder (*bool*, *optional*): + Skip any calculations for multimodal encoder. Useful if multimodal encoding is not going to be used. + image_attention_mask (`torch.Tensor` of shape `(batch_size, image_num_patches)`, *optional*): + Mask to avoid performing attention on padding pixel values for image inputs. Mask values selected in `[0, 1]`: + - 1 for pixel values that are real (i.e., **not masked**), + - 0 for pixel values that are padding (i.e., **masked**). Examples: @@ -1521,13 +1388,12 @@ class FlavaImageCodebookLayerGroup(nn.Module): # Inspired by DALLE Encoder in https://github.com/openai/DALL-E/blob/5be4b236bc3ade6943662354117a0e83752cc322/dall_e/encoder.py#L42 -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The FLAVA's image codebook model inspired from DALL-E's original encoder. Outputs raw hidden states and can be used to generate image tokens for an image based on DALL-E's vocab. Used to generate labels for MIM. Use `get_codebook_indices` to get image tokens for an image. - """, - FLAVA_START_DOCSTRING.format(config="FlavaImageCodebookConfig"), + """ ) class FlavaImageCodebook(FlavaPreTrainedModel): base_model_prefix = "" @@ -1738,11 +1604,10 @@ class FlavaGlobalContrastiveHead(nn.Module): return logits_per_image, logits_per_text, labels -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The FLAVA model for pretraining which outputs losses, embeddings, logits and transformer outputs. - """, - FLAVA_START_DOCSTRING.format(config="FlavaConfig") + FLAVA_PRETRAINING_START_DOCSTRING_EXTRA, + """ ) class FlavaForPreTraining(FlavaPreTrainedModel): # Those are linked to xxx.bias @@ -1754,6 +1619,11 @@ class FlavaForPreTraining(FlavaPreTrainedModel): ] def __init__(self, config: FlavaConfig, image_codebook: Optional[nn.Module] = None): + r""" + image_codebook ([`nn.Module`]): + If passed, the image codebook will be set to this. Otherwise, it will be initialized using the + image_codebook_config defined in the config first as the first parameter. + """ super().__init__(config) self.flava = FlavaModel(config) @@ -1788,10 +1658,7 @@ class FlavaForPreTraining(FlavaPreTrainedModel): x = x.view(x.size(0), -1) return x - @add_start_docstrings_to_model_forward( - FLAVA_PRETRAINING_INPUTS_DOCSTRING.format("batch_size, text_seq_len", "batch_size, image_num_patches") - ) - @replace_return_docstrings(output_type=FlavaForPreTrainingOutput, config_class=FlavaConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1812,7 +1679,52 @@ class FlavaForPreTraining(FlavaPreTrainedModel): return_dict: Optional[bool] = None, return_loss: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], FlavaForPreTrainingOutput]: - """ + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, text_seq_len)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + input_ids_masked (`torch.LongTensor` of shape `(batch_size, text_seq_len)`): + Indices of input sequence tokens in the vocabulary. These ones are the masked version of the original task + to be used with MLM. Indices can be obtained using [`AutoTokenizer`] along with + [`DataCollatorForMaskedLanguageModeling`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, text_seq_len)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + [What are token type IDs?](../glossary#token-type-ids) + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, image_num_patches)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + image_attention_mask (`torch.FloatTensor` of shape `(batch_size, image_num_patches)`, *optional*): + Mask to avoid performing attention on padding token indices specifically for images. Mask values selected + in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + [What are attention masks?](../glossary#attention-mask) + skip_unmasked_multimodal_encoder (*bool*, *optional*): + Skip any calculations for multimodal encoder for unmasked inputs. FLAVA pretraining doesn't need unmasked + multimodal embeddings or outputs as of now. + mlm_labels (`torch.LongTensor` of shape `(batch_size, text_seq_len)`, *optional*): + Labels for computing the left-to-right language and multimodal masked modeling loss (next word prediction). + Indices should be in `[-100, 0, ..., text_config.vocab_size - 1]` (see `input_ids` docstring). Tokens with + indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, + ..., text_config.vocab_size - 1]`. + mim_labels (`torch.LongTensor` of shape `(batch_size, image_num_patches)`, *optional*): + Labels for computing the image and multimodal masked modeling loss. Indices should be in `[-100, 0, ..., + image_config.vocab_size - 1]`. Tokens with indices set to `-100` are ignored (masked), the loss is only + computed for the tokens with labels in `[0, ..., image_config.vocab_size - 1]`. If not passed, they are + generated automatically using the image codebook assigned to the model. By default, it uses + [`FlavaImageCodebook`]. See [`FlavaImageCodebook`] to understand how to generate mim_labels. + itm_labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*): + Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match. + The pairs with 0 will be skipped for calculation of MMM and global contrastive losses as well. + return_loss (`bool`, *optional*, default to None): + Whether to return calculated loss or not. + codebook_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_image_patches, patch_size, patch_size, 3)`, *optional*): + Pixel values for image patches that are used to compute the image codebook labels for masked image modeling. + Examples: ```python >>> from PIL import Image @@ -1840,9 +1752,6 @@ class FlavaForPreTraining(FlavaPreTrainedModel): >>> output = model(**inputs) ``` - - Return: - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict return_loss = return_loss if return_loss is not None else self.config.return_loss diff --git a/src/transformers/models/fnet/modeling_fnet.py b/src/transformers/models/fnet/modeling_fnet.py index 63aaa42e9d..0a88a7b9fd 100755 --- a/src/transformers/models/fnet/modeling_fnet.py +++ b/src/transformers/models/fnet/modeling_fnet.py @@ -24,7 +24,7 @@ import torch.utils.checkpoint from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss -from ...utils import is_scipy_available +from ...utils import auto_docstring, is_scipy_available if is_scipy_available(): @@ -44,21 +44,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import logging from .configuration_fnet import FNetConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/fnet-base" -_CONFIG_FOR_DOC = "FNetConfig" - # Adapted from https://github.com/google-research/google-research/blob/master/f_net/fourier.py def _two_dim_matmul(x, matrix_dim_one, matrix_dim_two): @@ -395,12 +386,8 @@ class FNetPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class FNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FNetConfig base_model_prefix = "fnet" supports_gradient_checkpointing = True @@ -449,56 +436,7 @@ class FNetForPreTrainingOutput(ModelOutput): hidden_states: Optional[Tuple[torch.FloatTensor]] = None -FNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`FNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FNET_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare FNet Model transformer outputting raw hidden-states without any specific head on top.", - FNET_START_DOCSTRING, -) +@auto_docstring class FNetModel(FNetPreTrainedModel): """ @@ -508,6 +446,10 @@ class FNetModel(FNetPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -525,12 +467,7 @@ class FNetModel(FNetPreTrainedModel): def set_input_embeddings(self, value): self.embeddings.word_embeddings = value - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -601,12 +538,11 @@ class FNetModel(FNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" FNet Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. - """, - FNET_START_DOCSTRING, + """ ) class FNetForPreTraining(FNetPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -627,8 +563,7 @@ class FNetForPreTraining(FNetPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=FNetForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -651,10 +586,6 @@ class FNetForPreTraining(FNetPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. - - Returns: Example: @@ -702,7 +633,7 @@ class FNetForPreTraining(FNetPreTrainedModel): ) -@add_start_docstrings("""FNet Model with a `language modeling` head on top.""", FNET_START_DOCSTRING) +@auto_docstring class FNetForMaskedLM(FNetPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -722,12 +653,7 @@ class FNetForMaskedLM(FNetPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -770,9 +696,10 @@ class FNetForMaskedLM(FNetPreTrainedModel): return MaskedLMOutput(loss=masked_lm_loss, logits=prediction_scores, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """FNet Model with a `next sentence prediction (classification)` head on top.""", - FNET_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + FNet Model with a `next sentence prediction (classification)` head on top. + """ ) class FNetForNextSentencePrediction(FNetPreTrainedModel): def __init__(self, config): @@ -784,8 +711,7 @@ class FNetForNextSentencePrediction(FNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=NextSentencePredictorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -805,8 +731,6 @@ class FNetForNextSentencePrediction(FNetPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - Returns: - Example: ```python @@ -862,12 +786,11 @@ class FNetForNextSentencePrediction(FNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" FNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - FNET_START_DOCSTRING, + """ ) class FNetForSequenceClassification(FNetPreTrainedModel): def __init__(self, config): @@ -881,12 +804,7 @@ class FNetForSequenceClassification(FNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -947,13 +865,7 @@ class FNetForSequenceClassification(FNetPreTrainedModel): return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """ - FNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - FNET_START_DOCSTRING, -) +@auto_docstring class FNetForMultipleChoice(FNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -965,12 +877,7 @@ class FNetForMultipleChoice(FNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -982,6 +889,30 @@ class FNetForMultipleChoice(FNetPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1026,13 +957,7 @@ class FNetForMultipleChoice(FNetPreTrainedModel): return MultipleChoiceModelOutput(loss=loss, logits=reshaped_logits, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """ - FNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - FNET_START_DOCSTRING, -) +@auto_docstring class FNetForTokenClassification(FNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1046,12 +971,7 @@ class FNetForTokenClassification(FNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1095,13 +1015,7 @@ class FNetForTokenClassification(FNetPreTrainedModel): return TokenClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """ - FNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - FNET_START_DOCSTRING, -) +@auto_docstring class FNetForQuestionAnswering(FNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1114,12 +1028,7 @@ class FNetForQuestionAnswering(FNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1131,16 +1040,6 @@ class FNetForQuestionAnswering(FNetPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.fnet( diff --git a/src/transformers/models/focalnet/modeling_focalnet.py b/src/transformers/models/focalnet/modeling_focalnet.py index 3b9ec55dc8..37292f0687 100644 --- a/src/transformers/models/focalnet/modeling_focalnet.py +++ b/src/transformers/models/focalnet/modeling_focalnet.py @@ -27,31 +27,13 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_focalnet import FocalNetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "FocalNetConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/focalnet-tiny" -_EXPECTED_OUTPUT_SHAPE = [1, 49, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "microsoft/focalnet-tiny" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - @dataclass class FocalNetEncoderOutput(ModelOutput): @@ -621,12 +603,8 @@ class FocalNetEncoder(nn.Module): ) +@auto_docstring class FocalNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FocalNetConfig base_model_prefix = "focalnet" main_input_name = "pixel_values" @@ -653,37 +631,15 @@ class FocalNetPreTrainedModel(PreTrainedModel): module.gamma_2.data.fill_(self.config.layerscale_value) -FOCALNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`FocalNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FOCALNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare FocalNet Model outputting raw hidden-states without any specific head on top.", - FOCALNET_START_DOCSTRING, -) +@auto_docstring class FocalNetModel(FocalNetPreTrainedModel): def __init__(self, config, add_pooling_layer=True, use_mask_token=False): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config self.num_stages = len(config.depths) @@ -701,14 +657,7 @@ class FocalNetModel(FocalNetPreTrainedModel): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(FOCALNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=FocalNetModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -758,8 +707,9 @@ class FocalNetModel(FocalNetPreTrainedModel): ) -@add_start_docstrings( - """FocalNet Model with a decoder on top for masked image modeling. +@auto_docstring( + custom_intro=""" + FocalNet Model with a decoder on top for masked image modeling. This follows the same implementation as in [SimMIM](https://arxiv.org/abs/2111.09886). @@ -769,8 +719,7 @@ class FocalNetModel(FocalNetPreTrainedModel): directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - """, - FOCALNET_START_DOCSTRING, + """ ) class FocalNetForMaskedImageModeling(FocalNetPreTrainedModel): def __init__(self, config): @@ -790,8 +739,7 @@ class FocalNetForMaskedImageModeling(FocalNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FOCALNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=FocalNetMaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -803,8 +751,6 @@ class FocalNetForMaskedImageModeling(FocalNetPreTrainedModel): bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, FocalNetConfig, FocalNetForMaskedImageModeling @@ -873,12 +819,11 @@ class FocalNetForMaskedImageModeling(FocalNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" FocalNet Model with an image classification head on top (a linear layer on top of the pooled output) e.g. for ImageNet. - """, - FOCALNET_START_DOCSTRING, + """ ) class FocalNetForImageClassification(FocalNetPreTrainedModel): # Copied from transformers.models.swin.modeling_swin.SwinForImageClassification.__init__ with Swin->FocalNet, swin->focalnet @@ -896,13 +841,7 @@ class FocalNetForImageClassification(FocalNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FOCALNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=FocalNetImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -963,11 +902,10 @@ class FocalNetForImageClassification(FocalNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" FocalNet backbone, to be used with frameworks like X-Decoder. - """, - FOCALNET_START_DOCSTRING, + """ ) class FocalNetBackbone(FocalNetPreTrainedModel, BackboneMixin): def __init__(self, config: FocalNetConfig): @@ -980,17 +918,14 @@ class FocalNetBackbone(FocalNetPreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FOCALNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/fsmt/modeling_fsmt.py b/src/transformers/models/fsmt/modeling_fsmt.py index fb483d6a51..39f65179b8 100644 --- a/src/transformers/models/fsmt/modeling_fsmt.py +++ b/src/transformers/models/fsmt/modeling_fsmt.py @@ -44,21 +44,12 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_fsmt import FSMTConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/wmt19-ru-en" -_CONFIG_FOR_DOC = "FSMTConfig" # See all FSMT models at https://huggingface.co/models?filter=fsmt @@ -180,124 +171,6 @@ PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/w """ -FSMT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FSMTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. - -""" -FSMT_GENERATION_EXAMPLE = r""" - Translation example:: - - ```python - >>> from transformers import AutoTokenizer, FSMTForConditionalGeneration - - >>> mname = "facebook/wmt19-ru-en" - >>> model = FSMTForConditionalGeneration.from_pretrained(mname) - >>> tokenizer = AutoTokenizer.from_pretrained(mname) - - >>> src_text = "Машинное обучение - это здорово, не так ли?" - >>> input_ids = tokenizer(src_text, return_tensors="pt").input_ids - >>> outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3) - >>> tokenizer.decode(outputs[0], skip_special_tokens=True) - "Machine learning is great, isn't it?" - ``` - -""" - -FSMT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`FSTMTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`Tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden-states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`Tuple(torch.FloatTensor)` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*, defaults to `True`): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - def invert_mask(attention_mask): """Turns 1->0, 0->1, False->True, True-> False""" assert attention_mask.dim() == 2 @@ -341,6 +214,7 @@ def _prepare_fsmt_decoder_inputs( return decoder_input_ids, decoder_padding_mask, causal_mask +@auto_docstring class PretrainedFSMTModel(PreTrainedModel): config_class = FSMTConfig base_model_prefix = "model" @@ -1033,10 +907,7 @@ def _get_shape(t): return getattr(t, "shape", None) -@add_start_docstrings( - "The bare FSMT Model outputting raw hidden-states without any specific head on top.", - FSMT_START_DOCSTRING, -) +@auto_docstring class FSMTModel(PretrainedFSMTModel): _tied_weights_keys = ["decoder.embed_tokens.weight", "decoder.output_projection.weight"] @@ -1064,12 +935,7 @@ class FSMTModel(PretrainedFSMTModel): self._tie_or_clone_weights(self.decoder.embed_tokens, self.get_input_embeddings()) self._tie_or_clone_weights(self.decoder.output_projection, self.get_input_embeddings()) - @add_start_docstrings_to_model_forward(FSMT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: torch.LongTensor, @@ -1088,6 +954,27 @@ class FSMTModel(PretrainedFSMTModel): decoder_inputs_embeds: Optional[torch.FloatTensor] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ if decoder_input_ids is None: use_cache = False @@ -1175,8 +1062,10 @@ class FSMTModel(PretrainedFSMTModel): self.decoder.embed_tokens = value -@add_start_docstrings( - "The FSMT Model with a language modeling head. Can be used for summarization.", FSMT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The FSMT Model with a language modeling head. Can be used for summarization. + """ ) class FSMTForConditionalGeneration(PretrainedFSMTModel, GenerationMixin): base_model_prefix = "model" @@ -1190,9 +1079,7 @@ class FSMTForConditionalGeneration(PretrainedFSMTModel, GenerationMixin): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FSMT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(FSMT_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1213,13 +1100,45 @@ class FSMTForConditionalGeneration(PretrainedFSMTModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Translation: + ```python + >>> from transformers import AutoTokenizer, FSMTForConditionalGeneration + + >>> mname = "facebook/wmt19-ru-en" + >>> model = FSMTForConditionalGeneration.from_pretrained(mname) + >>> tokenizer = AutoTokenizer.from_pretrained(mname) + + >>> src_text = "Машинное обучение - это здорово, не так ли?" + >>> input_ids = tokenizer(src_text, return_tensors="pt").input_ids + >>> outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3) + >>> tokenizer.decode(outputs[0], skip_special_tokens=True) + "Machine learning is great, isn't it?" + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/funnel/modeling_funnel.py b/src/transformers/models/funnel/modeling_funnel.py index c5fecd0cc7..b77e802a20 100644 --- a/src/transformers/models/funnel/modeling_funnel.py +++ b/src/transformers/models/funnel/modeling_funnel.py @@ -33,22 +33,12 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_funnel import FunnelConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "FunnelConfig" -_CHECKPOINT_FOR_DOC = "funnel-transformer/small" - INF = 1e6 @@ -768,12 +758,8 @@ class FunnelDiscriminatorPredictions(nn.Module): return logits +@auto_docstring class FunnelPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = FunnelConfig load_tf_weights = load_tf_weights_in_funnel base_model_prefix = "funnel" @@ -846,70 +832,11 @@ class FunnelForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -FUNNEL_START_DOCSTRING = r""" - - The Funnel Transformer model was proposed in [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient - Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FunnelConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -FUNNEL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The base Funnel Transformer Model transformer outputting raw hidden-states without upsampling head (also called decoder) or any task-specific head on top. - """, - FUNNEL_START_DOCSTRING, + """ ) class FunnelBaseModel(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: @@ -927,12 +854,7 @@ class FunnelBaseModel(FunnelPreTrainedModel): def set_input_embeddings(self, new_embeddings: nn.Embedding) -> None: self.embeddings.word_embeddings = new_embeddings - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="funnel-transformer/small-base", - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -983,10 +905,7 @@ class FunnelBaseModel(FunnelPreTrainedModel): return encoder_outputs -@add_start_docstrings( - "The bare Funnel Transformer Model transformer outputting raw hidden-states without any specific head on top.", - FUNNEL_START_DOCSTRING, -) +@auto_docstring class FunnelModel(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: super().__init__(config) @@ -1004,12 +923,7 @@ class FunnelModel(FunnelPreTrainedModel): def set_input_embeddings(self, new_embeddings: nn.Embedding) -> None: self.embeddings.word_embeddings = new_embeddings - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1085,15 +999,12 @@ class FunnelModel(FunnelPreTrainedModel): ) -add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Funnel Transformer model with a binary classification head on top as used during pretraining for identifying generated tokens. - """, - FUNNEL_START_DOCSTRING, + """ ) - - class FunnelForPreTraining(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: super().__init__(config) @@ -1103,8 +1014,7 @@ class FunnelForPreTraining(FunnelPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=FunnelForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1124,8 +1034,6 @@ class FunnelForPreTraining(FunnelPreTrainedModel): - 0 indicates the token is an original token, - 1 indicates the token was replaced. - Returns: - Examples: ```python @@ -1176,7 +1084,7 @@ class FunnelForPreTraining(FunnelPreTrainedModel): ) -@add_start_docstrings("""Funnel Transformer Model with a `language modeling` head on top.""", FUNNEL_START_DOCSTRING) +@auto_docstring class FunnelForMaskedLM(FunnelPreTrainedModel): _tied_weights_keys = ["lm_head.weight"] @@ -1195,13 +1103,7 @@ class FunnelForMaskedLM(FunnelPreTrainedModel): def set_output_embeddings(self, new_embeddings: nn.Embedding) -> None: self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1251,12 +1153,11 @@ class FunnelForMaskedLM(FunnelPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Funnel Transformer Model with a sequence classification/regression head on top (two linear layer on top of the first timestep of the last hidden state) e.g. for GLUE tasks. - """, - FUNNEL_START_DOCSTRING, + """ ) class FunnelForSequenceClassification(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: @@ -1269,12 +1170,7 @@ class FunnelForSequenceClassification(FunnelPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="funnel-transformer/small-base", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1343,13 +1239,7 @@ class FunnelForSequenceClassification(FunnelPreTrainedModel): ) -@add_start_docstrings( - """ - Funnel Transformer Model with a multiple choice classification head on top (two linear layer on top of the first - timestep of the last hidden state, and a softmax) e.g. for RocStories/SWAG tasks. - """, - FUNNEL_START_DOCSTRING, -) +@auto_docstring class FunnelForMultipleChoice(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: super().__init__(config) @@ -1359,12 +1249,7 @@ class FunnelForMultipleChoice(FunnelPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint="funnel-transformer/small-base", - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1426,13 +1311,7 @@ class FunnelForMultipleChoice(FunnelPreTrainedModel): ) -@add_start_docstrings( - """ - Funnel Transformer Model with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - FUNNEL_START_DOCSTRING, -) +@auto_docstring class FunnelForTokenClassification(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: super().__init__(config) @@ -1445,12 +1324,7 @@ class FunnelForTokenClassification(FunnelPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1499,13 +1373,7 @@ class FunnelForTokenClassification(FunnelPreTrainedModel): ) -@add_start_docstrings( - """ - Funnel Transformer Model with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - FUNNEL_START_DOCSTRING, -) +@auto_docstring class FunnelForQuestionAnswering(FunnelPreTrainedModel): def __init__(self, config: FunnelConfig) -> None: super().__init__(config) @@ -1517,12 +1385,7 @@ class FunnelForQuestionAnswering(FunnelPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1535,16 +1398,6 @@ class FunnelForQuestionAnswering(FunnelPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.funnel( diff --git a/src/transformers/models/fuyu/modeling_fuyu.py b/src/transformers/models/fuyu/modeling_fuyu.py index a580d47ef1..ef3cd7309f 100644 --- a/src/transformers/models/fuyu/modeling_fuyu.py +++ b/src/transformers/models/fuyu/modeling_fuyu.py @@ -25,44 +25,14 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...models.auto.modeling_auto import AutoModel -from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, logging from .configuration_fuyu import FuyuConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "FuyuConfig" - -FUYU_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`FuyuConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Fuyu Model outputting raw hidden-states without any specific head on top.", - FUYU_START_DOCSTRING, -) +@auto_docstring class FuyuPreTrainedModel(PreTrainedModel): config_class = FuyuConfig base_model_prefix = "fuyu" @@ -86,81 +56,13 @@ class FuyuPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -FUYU_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - image_patches (`torch.FloatTensor` of shape `(batch_size, num_total_patches, patch_size_ x patch_size x num_channels)`, *optional*): - Image patches to be used as continuous embeddings. The patches are flattened and then projected to the - hidden size of the model. - image_patches_indices (`torch.LongTensor` of shape `(batch_size, num_total_patches + number_of_newline_tokens + number_of_text_tokens, patch_size_ x patch_size x num_channels )`, *optional*): - Indices indicating at which position the image_patches have to be inserted in input_embeds. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The Fuyu model which consists of a vision backbone and a language model, without a language modeling head.""", - FUYU_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Fuyu model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class FuyuModel(FuyuPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -226,7 +128,7 @@ class FuyuModel(FuyuPreTrainedModel): output_embeddings[batch_idx, dst_indices] = continuous_embeddings[batch_idx][src_indices] return output_embeddings - @add_start_docstrings_to_model_forward(FUYU_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -240,8 +142,15 @@ class FuyuModel(FuyuPreTrainedModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - **kwargs: Unpack[KwargsForCausalLM], + **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + image_patches (`torch.FloatTensor` of shape `(batch_size, num_total_patches, patch_size_ x patch_size x num_channels)`, *optional*): + Image patches to be used as continuous embeddings. The patches are flattened and then projected to the + hidden size of the model. + image_patches_indices (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Tensor of indices of the image patches in the input_ids tensor. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -303,9 +212,10 @@ class FuyuModel(FuyuPreTrainedModel): return outputs -@add_start_docstrings( - "Fuyu Model with a language modeling head on top for causal language model conditioned on image patches and text.", - FUYU_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Fuyu Model with a language modeling head on top for causal language model conditioned on image patches and text. + """ ) class FuyuForCausalLM(FuyuPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -340,8 +250,7 @@ class FuyuForCausalLM(FuyuPreTrainedModel, GenerationMixin): return self.model.get_decoder() @can_return_tuple - @add_start_docstrings_to_model_forward(FUYU_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -360,12 +269,15 @@ class FuyuForCausalLM(FuyuPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" + image_patches (`torch.FloatTensor` of shape `(batch_size, num_total_patches, patch_size_ x patch_size x num_channels)`, *optional*): + Image patches to be used as continuous embeddings. The patches are flattened and then projected to the + hidden size of the model. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. - - Returns: + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. + image_patches_indices (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Tensor of indices of the image patches in the input_ids tensor. Examples: diff --git a/src/transformers/models/gemma/modeling_gemma.py b/src/transformers/models/gemma/modeling_gemma.py index 555d8e22fc..7344bd0da0 100644 --- a/src/transformers/models/gemma/modeling_gemma.py +++ b/src/transformers/models/gemma/modeling_gemma.py @@ -39,16 +39,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_gemma import GemmaConfig @@ -60,9 +51,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/gemma-7b" -_CONFIG_FOR_DOC = "GemmaConfig" - class GemmaRMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): @@ -336,27 +324,7 @@ class GemmaDecoderLayer(GradientCheckpointingLayer): return outputs -GEMMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GemmaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Gemma Model outputting raw hidden-states without any specific head on top.", - GEMMA_START_DOCSTRING, -) +@auto_docstring class GemmaPreTrainedModel(PreTrainedModel): config_class = GemmaConfig base_model_prefix = "model" @@ -385,88 +353,8 @@ class GemmaPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GEMMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Gemma Model outputting raw hidden-states without any specific head on top.", - GEMMA_START_DOCSTRING, -) +@auto_docstring class GemmaModel(GemmaPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GemmaDecoderLayer`] - - Args: - config: GemmaConfig - """ - def __init__(self, config: GemmaConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -490,7 +378,7 @@ class GemmaModel(GemmaPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -714,6 +602,7 @@ class GemmaModel(GemmaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class GemmaForCausalLM(GemmaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -747,8 +636,7 @@ class GemmaForCausalLM(GemmaPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -765,19 +653,10 @@ class GemmaForCausalLM(GemmaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -832,8 +711,8 @@ class GemmaForCausalLM(GemmaPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Gemma Model transformer with a sequence classification head on top (linear layer). [`GemmaForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -844,8 +723,7 @@ class GemmaForCausalLM(GemmaPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GEMMA_START_DOCSTRING, + """ ) class GemmaForSequenceClassification(GemmaPreTrainedModel): def __init__(self, config): @@ -864,7 +742,7 @@ class GemmaForSequenceClassification(GemmaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -933,13 +811,7 @@ class GemmaForSequenceClassification(GemmaPreTrainedModel): ) -@add_start_docstrings( - """ - The Gemma Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - GEMMA_START_DOCSTRING, -) +@auto_docstring class GemmaForTokenClassification(GemmaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -964,12 +836,7 @@ class GemmaForTokenClassification(GemmaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/gemma/modular_gemma.py b/src/transformers/models/gemma/modular_gemma.py index ec16f5bba8..fd7bae3855 100644 --- a/src/transformers/models/gemma/modular_gemma.py +++ b/src/transformers/models/gemma/modular_gemma.py @@ -42,7 +42,6 @@ VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} SPIECE_UNDERLINE = "▁" -_CHECKPOINT_FOR_DOC = "google/gemma-7b" logger = logging.get_logger(__name__) @@ -467,20 +466,6 @@ class GemmaModel(LlamaModel): class GemmaForCausalLM(LlamaForCausalLM): def forward(**super_kwargs): r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: - Example: ```python diff --git a/src/transformers/models/gemma2/modeling_gemma2.py b/src/transformers/models/gemma2/modeling_gemma2.py index a1a308f1d8..04f3edbf6e 100644 --- a/src/transformers/models/gemma2/modeling_gemma2.py +++ b/src/transformers/models/gemma2/modeling_gemma2.py @@ -38,15 +38,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from ...utils.deprecation import deprecate_kwarg from .configuration_gemma2 import Gemma2Config @@ -60,10 +52,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/gemma2-7b" -_CONFIG_FOR_DOC = "Gemma2Config" - - class Gemma2RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() @@ -393,27 +381,7 @@ class Gemma2RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -GEMMA2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Gemma2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Gemma2 Model outputting raw hidden-states without any specific head on top.", - GEMMA2_START_DOCSTRING, -) +@auto_docstring class Gemma2PreTrainedModel(PreTrainedModel): config_class = Gemma2Config base_model_prefix = "model" @@ -442,88 +410,8 @@ class Gemma2PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GEMMA2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Gemma2 Model outputting raw hidden-states without any specific head on top.", - GEMMA2_START_DOCSTRING, -) +@auto_docstring class Gemma2Model(Gemma2PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Gemma2DecoderLayer`] - - Args: - config: Gemma2Config - """ - def __init__(self, config: Gemma2Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -547,8 +435,7 @@ class Gemma2Model(Gemma2PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA2_INPUTS_DOCSTRING) - @deprecate_kwarg("last_cache_position", version="4.53.0") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -761,6 +648,7 @@ class Gemma2Model(Gemma2PreTrainedModel): return causal_mask +@auto_docstring class Gemma2ForCausalLM(Gemma2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -794,8 +682,7 @@ class Gemma2ForCausalLM(Gemma2PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -812,19 +699,10 @@ class Gemma2ForCausalLM(Gemma2PreTrainedModel, GenerationMixin): **loss_kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -942,8 +820,8 @@ class Gemma2ForCausalLM(Gemma2PreTrainedModel, GenerationMixin): return model_inputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Gemma2 Model transformer with a sequence classification head on top (linear layer). [`Gemma2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -954,8 +832,7 @@ class Gemma2ForCausalLM(Gemma2PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GEMMA2_START_DOCSTRING, + """ ) class Gemma2ForSequenceClassification(Gemma2PreTrainedModel): def __init__(self, config): @@ -974,7 +851,7 @@ class Gemma2ForSequenceClassification(Gemma2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1043,13 +920,7 @@ class Gemma2ForSequenceClassification(Gemma2PreTrainedModel): ) -@add_start_docstrings( - """ - The Gemma2 Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - GEMMA2_START_DOCSTRING, -) +@auto_docstring class Gemma2ForTokenClassification(Gemma2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1074,12 +945,7 @@ class Gemma2ForTokenClassification(Gemma2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/gemma2/modular_gemma2.py b/src/transformers/models/gemma2/modular_gemma2.py index 5234938dc7..9a448370dd 100644 --- a/src/transformers/models/gemma2/modular_gemma2.py +++ b/src/transformers/models/gemma2/modular_gemma2.py @@ -27,7 +27,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import add_start_docstrings_to_model_forward, can_return_tuple, is_torch_flex_attn_available, logging +from ...utils import is_torch_flex_attn_available, logging from ...utils.deprecation import deprecate_kwarg from ..gemma.modeling_gemma import ( GemmaAttention, @@ -42,10 +42,6 @@ from ..gemma.modeling_gemma import ( ) -_CHECKPOINT_FOR_DOC = "google/gemma2-7b" -GEMMA2_INPUTS_DOCSTRING = None # Will be picked up by modular - - if is_torch_flex_attn_available(): from torch.nn.attention.flex_attention import BlockMask @@ -413,9 +409,6 @@ class Gemma2Model(GemmaModel): [Gemma2DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] ) - @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA2_INPUTS_DOCSTRING) - @deprecate_kwarg("last_cache_position", version="4.53.0") def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -595,20 +588,6 @@ class Gemma2ForCausalLM(GemmaForCausalLM): **loss_kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: - Example: ```python diff --git a/src/transformers/models/gemma3/image_processing_gemma3_fast.py b/src/transformers/models/gemma3/image_processing_gemma3_fast.py index 7a18d456f3..d7a5e87e01 100644 --- a/src/transformers/models/gemma3/image_processing_gemma3_fast.py +++ b/src/transformers/models/gemma3/image_processing_gemma3_fast.py @@ -19,8 +19,6 @@ import math from typing import List, Optional, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -36,7 +34,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -61,26 +59,24 @@ logger = logging.get_logger(__name__) class Gemma3FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + do_pan_and_scan (`bool`, *optional*): + Whether to apply `pan_and_scan` to images. + pan_and_scan_min_crop_size (`int`, *optional*): + Minimum size of each crop in pan and scan. + pan_and_scan_max_num_crops (`int`, *optional*): + Maximum number of crops per image in pan and scan. + pan_and_scan_min_ratio_to_activate (`float`, *optional*): + Minimum aspect ratio to activate pan and scan. + """ + do_pan_and_scan: Optional[bool] pan_and_scan_min_crop_size: Optional[int] pan_and_scan_max_num_crops: Optional[int] pan_and_scan_min_ratio_to_activate: Optional[float] -@add_start_docstrings( - "Constructs a fast ConvNeXT image processor. Based on [`SiglipImageProcessor`] with incorporation of Pan adn Scan cropping method.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - do_pan_and_scan (`bool`, *optional*): - Whether to apply `pan_and_scan` to images. - pan_and_scan_min_crop_size (`int`, *optional*): - Minimum size of each crop in pan and scan. - pan_and_scan_max_num_crops (`int`, *optional*): - Maximum number of crops per image in pan and scan. - pan_and_scan_min_ratio_to_activate (`float`, *optional*): - Minimum aspect ratio to activate pan and scan. - """, -) +@auto_docstring class Gemma3ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN @@ -184,19 +180,7 @@ class Gemma3ImageProcessorFast(BaseImageProcessorFast): num_crops = [len(pas_images) for _ in images] return pas_images, num_crops - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - do_pan_and_scan (`bool`, *optional*): - Whether to apply `pan_and_scan` to images. - pan_and_scan_min_crop_size (`int`, *optional*): - Minimum size of each crop in pan and scan. - pan_and_scan_max_num_crops (`int`, *optional*): - Maximum number of crops per image in pan and scan. - pan_and_scan_min_ratio_to_activate (`float`, *optional*): - Minimum aspect ratio to activate pan and scan. - """, - ) + @auto_docstring def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index 74642ea6ba..d9fbe9a41a 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -38,13 +38,11 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, is_torch_flex_attn_available, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from ...utils.deprecation import deprecate_kwarg from ..auto import AutoModel @@ -58,7 +56,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Gemma3Config" @dataclass @@ -491,27 +488,7 @@ class Gemma3DecoderLayer(nn.Module): return outputs -GEMMA3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Gemma3Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Gemma3 Model outputting raw hidden-states without any specific head on top.", - GEMMA3_START_DOCSTRING, -) +@auto_docstring class Gemma3PreTrainedModel(PreTrainedModel): config_class = Gemma3Config base_model_prefix = "" @@ -548,88 +525,8 @@ class Gemma3PreTrainedModel(PreTrainedModel): module.mm_input_projection_weight.data.zero_() -GEMMA3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Gemma3Text Model outputting raw hidden-states without any specific head on top.", - GEMMA3_START_DOCSTRING, -) +@auto_docstring class Gemma3TextModel(Gemma3PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Gemma3TextDecoderLayer`] - - Args: - config: Gemma3TextConfig - """ - config_class = Gemma3TextConfig def __init__(self, config: Gemma3TextConfig): @@ -665,8 +562,7 @@ class Gemma3TextModel(Gemma3PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) - @deprecate_kwarg("last_cache_position", version="4.53.0") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -880,6 +776,7 @@ class Gemma3TextModel(Gemma3PreTrainedModel): return causal_mask +@auto_docstring class Gemma3ForCausalLM(Gemma3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -915,8 +812,7 @@ class Gemma3ForCausalLM(Gemma3PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -933,19 +829,10 @@ class Gemma3ForCausalLM(Gemma3PreTrainedModel, GenerationMixin): **loss_kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1099,9 +986,10 @@ class Gemma3MultiModalProjector(nn.Module): return projected_vision_outputs.type_as(vision_outputs) -@add_start_docstrings( - """Base Gemma3 model which consists of a vision backbone and a language model withou language modeling head.""", - GEMMA3_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Base Gemma3 model which consists of a vision backbone and a language model withou language modeling head., + """ ) class Gemma3Model(Gemma3PreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -1208,7 +1096,7 @@ class Gemma3Model(Gemma3PreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1226,6 +1114,33 @@ class Gemma3Model(Gemma3PreTrainedModel): return_dict: Optional[bool] = None, **lm_kwargs, ) -> Union[Tuple, Gemma3ModelOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration + + >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma32-3b-mix-224") + >>> processor = AutoProcessor.from_pretrained("google/gemma32-3b-mix-224") + + >>> prompt = "Where is the cat standing?" + >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, text=prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(**inputs,) + >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Where is the cat standing?\nsnow" + ```""" if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -1301,9 +1216,10 @@ class Gemma3Model(Gemma3PreTrainedModel): ) -@add_start_docstrings( - """The Gemma3 model which consists of a vision backbone and a language model.""", - GEMMA3_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Base Gemma3 model which consists of a vision backbone and a language model without language modeling head., + """ ) class Gemma3ForConditionalGeneration(Gemma3PreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -1345,8 +1261,7 @@ class Gemma3ForConditionalGeneration(Gemma3PreTrainedModel, GenerationMixin): def multi_modal_projector(self): return self.model.multi_modal_projector - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Gemma3CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1366,19 +1281,10 @@ class Gemma3ForConditionalGeneration(Gemma3PreTrainedModel, GenerationMixin): **lm_kwargs, ) -> Union[Tuple, Gemma3CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. Example: diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 24206e6f83..ab1db5eb74 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -30,14 +30,7 @@ from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import rope_config_validation from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging from ...utils.deprecation import deprecate_kwarg from ..gemma2.configuration_gemma2 import Gemma2Config from ..gemma2.modeling_gemma2 import ( @@ -60,13 +53,8 @@ from ..paligemma.modeling_paligemma import ( from ..siglip import SiglipVisionConfig -_CHECKPOINT_FOR_DOC = "google/gemma-3-4b" -_CONFIG_FOR_DOC = "Gemma3Config" - logger = logging.get_logger(__name__) -GEMMA3_INPUTS_DOCSTRING = None # Will be picked up by modular - class Gemma3TextConfig(Gemma2Config): r""" @@ -564,9 +552,6 @@ class Gemma3TextModel(Gemma2Model): config.rope_scaling = {"rope_type": "default"} self.rotary_emb_local = Gemma3RotaryEmbedding(config=config) - @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) - @deprecate_kwarg("last_cache_position", version="4.53.0") def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -816,7 +801,7 @@ class Gemma3Model(PaliGemmaModel): return causal_mask @can_return_tuple - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -909,13 +894,8 @@ class Gemma3Model(PaliGemmaModel): ) -@add_start_docstrings( - """The Gemma3 model which consists of a vision backbone and a language model.""", - GEMMA3_START_DOCSTRING, -) class Gemma3ForConditionalGeneration(PaliGemmaForConditionalGeneration): - @add_start_docstrings_to_model_forward(GEMMA3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Gemma3CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -935,19 +915,10 @@ class Gemma3ForConditionalGeneration(PaliGemmaForConditionalGeneration): **lm_kwargs, ) -> Union[Tuple, Gemma3CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. Example: diff --git a/src/transformers/models/git/modeling_git.py b/src/transformers/models/git/modeling_git.py index bf5934c644..c63b00fe6b 100644 --- a/src/transformers/models/git/modeling_git.py +++ b/src/transformers/models/git/modeling_git.py @@ -37,10 +37,8 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, torch_int, ) from .configuration_git import GitConfig, GitVisionConfig @@ -48,9 +46,6 @@ from .configuration_git import GitConfig, GitVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/git-base" -_CONFIG_FOR_DOC = "GitConfig" - @dataclass # Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->Git @@ -497,12 +492,8 @@ class GitEncoder(nn.Module): ) +@auto_docstring class GitPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GitConfig base_model_prefix = "git" supports_gradient_checkpointing = True @@ -530,90 +521,6 @@ class GitPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GIT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GitConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GIT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`CLIPImageProcessor.__call__`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.clip.modeling_clip.CLIPVisionEmbeddings with CLIP->Git class GitVisionEmbeddings(nn.Module): def __init__(self, config: GitVisionConfig): @@ -964,24 +871,6 @@ class GitVisionEncoder(nn.Module): ) -GIT_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class GitVisionTransformer(nn.Module): # Copied from transformers.models.altclip.modeling_altclip.AltCLIPVisionTransformer.__init__ with AltCLIPEncoder->GitVisionEncoder, AltCLIP->Git def __init__(self, config: GitVisionConfig): @@ -994,8 +883,7 @@ class GitVisionTransformer(nn.Module): self.encoder = GitVisionEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(GIT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=GitVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1004,10 +892,6 @@ class GitVisionTransformer(nn.Module): interpolate_pos_encoding: Optional[bool] = False, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1041,9 +925,10 @@ class GitVisionTransformer(nn.Module): ) -@add_start_docstrings( - """The vision model from CLIP, used in GIT, without any head or projection on top.""", - GIT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from CLIP, used in GIT, without any head or projection on top. + """ ) class GitVisionModel(GitPreTrainedModel): config_class = GitVisionConfig @@ -1059,8 +944,7 @@ class GitVisionModel(GitPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(GIT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=GitVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1070,8 +954,6 @@ class GitVisionModel(GitPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: r""" - Returns: - Examples: ```python @@ -1114,10 +996,10 @@ class GitProjection(nn.Module): return self.visual_projection(embeddings) -@add_start_docstrings( - "The bare GIT Model transformer consisting of a CLIP image encoder and text decoder outputting raw hidden-states" - " without any specific head on top.", - GIT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare GIT Model transformer consisting of a CLIP image encoder and text decoder outputting raw hidden-states + """ ) class GitModel(GitPreTrainedModel): def __init__(self, config): @@ -1209,8 +1091,7 @@ class GitModel(GitPreTrainedModel): return full_attention_mask - @add_start_docstrings_to_model_forward(GIT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1227,12 +1108,6 @@ class GitModel(GitPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPooling]: r""" - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: - Examples: ```python @@ -1383,8 +1258,10 @@ class GitModel(GitPreTrainedModel): ) -@add_start_docstrings( - """GIT Model with a `language modeling` head on top for autoregressive language modeling.""", GIT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + GIT Model with a `language modeling` head on top for autoregressive language modeling. + """ ) class GitForCausalLM(GitPreTrainedModel, GenerationMixin): _tied_weights_keys = ["output.weight"] @@ -1404,8 +1281,7 @@ class GitForCausalLM(GitPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.output = new_embeddings - @add_start_docstrings_to_model_forward(GIT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1428,11 +1304,6 @@ class GitForCausalLM(GitPreTrainedModel, GenerationMixin): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Examples: diff --git a/src/transformers/models/glm/modeling_glm.py b/src/transformers/models/glm/modeling_glm.py index 38e0282264..607a3b7f69 100644 --- a/src/transformers/models/glm/modeling_glm.py +++ b/src/transformers/models/glm/modeling_glm.py @@ -40,16 +40,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_glm import GlmConfig @@ -61,9 +52,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "THUDM/glm-4-9b" -_CONFIG_FOR_DOC = "GlmConfig" - class GlmMLP(nn.Module): def __init__(self, config): @@ -353,27 +341,7 @@ class GlmDecoderLayer(GradientCheckpointingLayer): return outputs -GLM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GlmConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Glm Model outputting raw hidden-states without any specific head on top.", - GLM_START_DOCSTRING, -) +@auto_docstring class GlmPreTrainedModel(PreTrainedModel): config_class = GlmConfig base_model_prefix = "model" @@ -402,88 +370,8 @@ class GlmPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GLM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Glm Model outputting raw hidden-states without any specific head on top.", - GLM_START_DOCSTRING, -) +@auto_docstring class GlmModel(GlmPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GlmDecoderLayer`] - - Args: - config: GlmConfig - """ - def __init__(self, config: GlmConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -507,7 +395,7 @@ class GlmModel(GlmPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -729,6 +617,7 @@ class GlmModel(GlmPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class GlmForCausalLM(GlmPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -762,8 +651,7 @@ class GlmForCausalLM(GlmPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -780,19 +668,10 @@ class GlmForCausalLM(GlmPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -847,8 +726,8 @@ class GlmForCausalLM(GlmPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Glm Model transformer with a sequence classification head on top (linear layer). [`GlmForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -859,8 +738,7 @@ class GlmForCausalLM(GlmPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GLM_START_DOCSTRING, + """ ) class GlmForSequenceClassification(GlmPreTrainedModel): def __init__(self, config): @@ -879,7 +757,7 @@ class GlmForSequenceClassification(GlmPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -948,13 +826,7 @@ class GlmForSequenceClassification(GlmPreTrainedModel): ) -@add_start_docstrings( - """ - The Glm Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - GLM_START_DOCSTRING, -) +@auto_docstring class GlmForTokenClassification(GlmPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -979,12 +851,7 @@ class GlmForTokenClassification(GlmPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/glm4/modeling_glm4.py b/src/transformers/models/glm4/modeling_glm4.py index 0d7ac7c769..98ee428e2c 100644 --- a/src/transformers/models/glm4/modeling_glm4.py +++ b/src/transformers/models/glm4/modeling_glm4.py @@ -40,16 +40,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_glm4 import Glm4Config @@ -61,9 +52,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "THUDM/GLM-4-9B-Chat-0414" -_CONFIG_FOR_DOC = "Glm4Config" - class Glm4MLP(nn.Module): def __init__(self, config): @@ -361,27 +349,7 @@ class Glm4RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -GLM4_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Glm4Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Glm4 Model outputting raw hidden-states without any specific head on top.", - GLM4_START_DOCSTRING, -) +@auto_docstring class Glm4PreTrainedModel(PreTrainedModel): config_class = Glm4Config base_model_prefix = "model" @@ -410,88 +378,8 @@ class Glm4PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GLM4_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Glm4 Model outputting raw hidden-states without any specific head on top.", - GLM4_START_DOCSTRING, -) +@auto_docstring class Glm4Model(Glm4PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Glm4DecoderLayer`] - - Args: - config: Glm4Config - """ - def __init__(self, config: Glm4Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -515,7 +403,7 @@ class Glm4Model(Glm4PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GLM4_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -734,6 +622,7 @@ class Glm4Model(Glm4PreTrainedModel): return causal_mask +@auto_docstring class Glm4ForCausalLM(Glm4PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -767,8 +656,7 @@ class Glm4ForCausalLM(Glm4PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(GLM4_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -785,19 +673,10 @@ class Glm4ForCausalLM(Glm4PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -852,8 +731,8 @@ class Glm4ForCausalLM(Glm4PreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Glm4 Model transformer with a sequence classification head on top (linear layer). [`Glm4ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -864,8 +743,7 @@ class Glm4ForCausalLM(Glm4PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GLM4_START_DOCSTRING, + """ ) class Glm4ForSequenceClassification(Glm4PreTrainedModel): def __init__(self, config): @@ -884,7 +762,7 @@ class Glm4ForSequenceClassification(Glm4PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GLM4_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -953,13 +831,7 @@ class Glm4ForSequenceClassification(Glm4PreTrainedModel): ) -@add_start_docstrings( - """ - The Glm4 Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - GLM4_START_DOCSTRING, -) +@auto_docstring class Glm4ForTokenClassification(Glm4PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -984,12 +856,7 @@ class Glm4ForTokenClassification(Glm4PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GLM4_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/glm4/modular_glm4.py b/src/transformers/models/glm4/modular_glm4.py index 2a52e5b670..4664efcad2 100644 --- a/src/transformers/models/glm4/modular_glm4.py +++ b/src/transformers/models/glm4/modular_glm4.py @@ -23,12 +23,7 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import CausalLMOutputWithPast from ...processing_utils import Unpack from ...utils import LossKwargs, logging -from ..glm.modeling_glm import ( - GlmAttention, - GlmForCausalLM, - GlmForSequenceClassification, - GlmForTokenClassification, -) +from ..glm.modeling_glm import GlmAttention, GlmForCausalLM, GlmForSequenceClassification, GlmForTokenClassification from ..phi3.modeling_phi3 import Phi3MLP from .configuration_glm4 import Glm4Config from .modeling_glm4 import Glm4RMSNorm @@ -114,19 +109,10 @@ class Glm4ForCausalLM(GlmForCausalLM): **super_kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/glpn/modeling_glpn.py b/src/transformers/models/glpn/modeling_glpn.py index 0c90746d18..b9bb0dba2a 100755 --- a/src/transformers/models/glpn/modeling_glpn.py +++ b/src/transformers/models/glpn/modeling_glpn.py @@ -25,27 +25,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, DepthEstimatorOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_glpn import GLPNConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "GLPNConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "vinvino02/glpn-kitti" -_EXPECTED_OUTPUT_SHAPE = [1, 512, 15, 20] - - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: """ @@ -413,12 +399,8 @@ class GLPNEncoder(nn.Module): ) +@auto_docstring class GLPNPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GLPNConfig base_model_prefix = "glpn" main_input_name = "pixel_values" @@ -442,39 +424,7 @@ class GLPNPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GLPN_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`GLPNConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GLPN_INPUTS_DOCSTRING = r""" - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`GLPNImageProcessor.__call__`] for details. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare GLPN encoder (Mix-Transformer) outputting raw hidden-states without any specific head on top.", - GLPN_START_DOCSTRING, -) +@auto_docstring class GLPNModel(GLPNPreTrainedModel): # Copied from transformers.models.segformer.modeling_segformer.SegformerModel.__init__ with Segformer->GLPN def __init__(self, config): @@ -495,14 +445,7 @@ class GLPNModel(GLPNPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(GLPN_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring # Copied from transformers.models.segformer.modeling_segformer.SegformerModel.forward def forward( self, @@ -672,9 +615,10 @@ class GLPNDepthEstimationHead(nn.Module): return predicted_depth -@add_start_docstrings( - """GLPN Model transformer with a lightweight depth estimation head on top e.g. for KITTI, NYUv2.""", - GLPN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + GLPN Model transformer with a lightweight depth estimation head on top e.g. for KITTI, NYUv2. + """ ) class GLPNForDepthEstimation(GLPNPreTrainedModel): def __init__(self, config): @@ -687,8 +631,7 @@ class GLPNForDepthEstimation(GLPNPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GLPN_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=DepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -701,8 +644,6 @@ class GLPNForDepthEstimation(GLPNPreTrainedModel): labels (`torch.FloatTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth depth estimation maps for computing the loss. - Returns: - Examples: ```python diff --git a/src/transformers/models/got_ocr2/image_processing_got_ocr2_fast.py b/src/transformers/models/got_ocr2/image_processing_got_ocr2_fast.py index e8b17c4ed3..3b8b4c2f56 100644 --- a/src/transformers/models/got_ocr2/image_processing_got_ocr2_fast.py +++ b/src/transformers/models/got_ocr2/image_processing_got_ocr2_fast.py @@ -18,8 +18,6 @@ from typing import List, Optional, Tuple, Union from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -35,7 +33,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -54,26 +52,24 @@ if is_torchvision_available(): class GotOcr2ImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + crop_to_patches (`bool`, *optional*, defaults to `False`): + Whether to crop the image to patches. Can be overridden by the `crop_to_patches` parameter in the + `preprocess` method. + min_patches (`int`, *optional*, defaults to 1): + The minimum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is + set to `True`. Can be overridden by the `min_patches` parameter in the `preprocess` method. + max_patches (`int`, *optional*, defaults to 12): + The maximum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is + set to `True`. Can be overridden by the `max_patches` parameter in the `preprocess` method. + """ + crop_to_patches: Optional[bool] min_patches: Optional[int] max_patches: Optional[int] -@add_start_docstrings( - "Constructs a fast GotOcr2 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - crop_to_patches (`bool`, *optional*, defaults to `False`): - Whether to crop the image to patches. Can be overridden by the `crop_to_patches` parameter in the - `preprocess` method. - min_patches (`int`, *optional*, defaults to 1): - The minimum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is - set to `True`. Can be overridden by the `min_patches` parameter in the `preprocess` method. - max_patches (`int`, *optional*, defaults to 12): - The maximum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is - set to `True`. Can be overridden by the `max_patches` parameter in the `preprocess` method. - """, -) +@auto_docstring class GotOcr2ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN @@ -88,24 +84,11 @@ class GotOcr2ImageProcessorFast(BaseImageProcessorFast): max_patches = 12 valid_kwargs = GotOcr2ImageProcessorKwargs - def __init__(self, **kwargs: Unpack[valid_kwargs]): + def __init__(self, **kwargs: Unpack[GotOcr2ImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - crop_to_patches (`bool`, *optional*, defaults to `False`): - Whether to crop the image to patches. Can be overridden by the `crop_to_patches` parameter in the - `preprocess` method. - min_patches (`int`, *optional*, defaults to 1): - The minimum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is - set to `True`. Can be overridden by the `min_patches` parameter in the `preprocess` method. - max_patches (`int`, *optional*, defaults to 12): - The maximum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is - set to `True`. Can be overridden by the `max_patches` parameter in the `preprocess` method. - """, - ) - def preprocess(self, images: ImageInput, **kwargs: Unpack[valid_kwargs]) -> BatchFeature: + @auto_docstring + def preprocess(self, images: ImageInput, **kwargs: Unpack[GotOcr2ImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) def crop_image_to_patches( diff --git a/src/transformers/models/got_ocr2/modeling_got_ocr2.py b/src/transformers/models/got_ocr2/modeling_got_ocr2.py index 145a9ce752..f99598c4dc 100644 --- a/src/transformers/models/got_ocr2/modeling_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modeling_got_ocr2.py @@ -34,20 +34,11 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple from ..auto import AutoModel from .configuration_got_ocr2 import GotOcr2Config, GotOcr2VisionConfig -_CONFIG_FOR_DOC = "GotOcr2Config" - - class GotOcr2MLPBlock(nn.Module): def __init__(self, config): super().__init__() @@ -591,27 +582,7 @@ class GotOcr2ModelOutputWithPast(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None -GOT_OCR2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GotOcr2Config`] or [`GotOcr2VisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GotOcr2 Model outputting raw hidden-states without any specific head on top.", - GOT_OCR2_START_DOCSTRING, -) +@auto_docstring class GotOcr2PreTrainedModel(PreTrainedModel): config_class = GotOcr2Config base_model_prefix = "" @@ -643,79 +614,10 @@ class GotOcr2PreTrainedModel(PreTrainedModel): module.pos_embed.data.zero_() -GOT_OCR2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(seq_length, num_channels * image_size * image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`GotOcr2ImageProcessor.__call__`] for details. [`GotOcr2Processor`] uses - [`GotOcr2ImageProcessor`] for processing images. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The GotOcr2 model which consists of a vision backbone and a language model, without a language modeling head.""", - GOT_OCR2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The GotOcr2 model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class GotOcr2Model(GotOcr2PreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -750,7 +652,7 @@ class GotOcr2Model(GotOcr2PreTrainedModel): return self.multi_modal_projector(image_outputs) @can_return_tuple - @add_start_docstrings_to_model_forward(GOT_OCR2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -821,9 +723,10 @@ class GotOcr2Model(GotOcr2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The GOT_OCR2 model which consists of a vision backbone and a language model.""", - GOT_OCR2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The GOT_OCR2 model which consists of a vision backbone and a language model. + """ ) class GotOcr2ForConditionalGeneration(GotOcr2PreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -866,8 +769,7 @@ class GotOcr2ForConditionalGeneration(GotOcr2PreTrainedModel, GenerationMixin): return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(GOT_OCR2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GotOcr2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -886,20 +788,10 @@ class GotOcr2ForConditionalGeneration(GotOcr2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, GotOcr2CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/got_ocr2/modular_got_ocr2.py b/src/transformers/models/got_ocr2/modular_got_ocr2.py index fe02b61477..80a2a23c2f 100644 --- a/src/transformers/models/got_ocr2/modular_got_ocr2.py +++ b/src/transformers/models/got_ocr2/modular_got_ocr2.py @@ -33,13 +33,7 @@ from transformers.models.sam.modeling_sam import SamMLPBlock, SamVisionAttention from ...configuration_utils import PretrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings_to_model_forward, - can_return_tuple, - is_vision_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_vision_available, logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -48,8 +42,6 @@ if is_vision_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GotOcr2Config" - class GotOcr2VisionConfig(PretrainedConfig): r""" @@ -306,76 +298,6 @@ class GotOcr2PreTrainedModel(LlavaPreTrainedModel): module.pos_embed.data.zero_() -GOT_OCR2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(seq_length, num_channels * image_size * image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`GotOcr2ImageProcessor.__call__`] for details. [`GotOcr2Processor`] uses - [`GotOcr2ImageProcessor`] for processing images. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class GotOcr2Model(LlavaModel): def __init__(self, config: GotOcr2Config): super().__init__(config) @@ -396,8 +318,6 @@ class GotOcr2Model(LlavaModel): image_outputs = self.vision_tower(pixel_values).last_hidden_state return self.multi_modal_projector(image_outputs) - @can_return_tuple - @add_start_docstrings_to_model_forward(GOT_OCR2_INPUTS_DOCSTRING) def forward( self, input_ids: torch.LongTensor = None, @@ -467,8 +387,7 @@ class GotOcr2Model(LlavaModel): class GotOcr2ForConditionalGeneration(LlavaForConditionalGeneration): @can_return_tuple - @add_start_docstrings_to_model_forward(GOT_OCR2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GotOcr2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -487,20 +406,10 @@ class GotOcr2ForConditionalGeneration(LlavaForConditionalGeneration): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, GotOcr2CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/gpt2/modeling_gpt2.py b/src/transformers/models/gpt2/modeling_gpt2.py index 0af4d99065..4b3853b436 100644 --- a/src/transformers/models/gpt2/modeling_gpt2.py +++ b/src/transformers/models/gpt2/modeling_gpt2.py @@ -41,11 +41,9 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import Conv1D, find_pruneable_heads_and_indices, prune_conv1d_layer from ...utils import ( ModelOutput, - add_code_sample_docstrings, add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from ...utils.deprecation import deprecate_kwarg from ...utils.model_parallel_utils import assert_device_map, get_device_map @@ -54,9 +52,6 @@ from .configuration_gpt2 import GPT2Config logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "openai-community/gpt2" -_CONFIG_FOR_DOC = "GPT2Config" - def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path): """Load tf checkpoints in a pytorch model""" @@ -554,12 +549,8 @@ class GPT2SequenceSummary(nn.Module): return output +@auto_docstring class GPT2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GPT2Config load_tf_weights = load_tf_weights_in_gpt2 base_model_prefix = "transformer" @@ -646,107 +637,6 @@ class GPT2DoubleHeadsModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -GPT2_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GPT2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GPT2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else - `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input - sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If `past_key_values` is used, `attention_mask` needs to contain the masking strategy that was used for - `past_key_values`. In other words, the `attention_mask` always has to have the length: - `len(past_key_values) + len(input_ids)` - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, input_ids_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" PARALLELIZE_DOCSTRING = r""" This is an experimental feature and is a subject to change at a moment's notice. @@ -799,10 +689,7 @@ DEPARALLELIZE_DOCSTRING = r""" """ -@add_start_docstrings( - "The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.", - GPT2_START_DOCSTRING, -) +@auto_docstring class GPT2Model(GPT2PreTrainedModel): _supports_param_buffer_assignment = False @@ -884,12 +771,7 @@ class GPT2Model(GPT2PreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].attn.prune_heads(heads) - @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -908,6 +790,20 @@ class GPT2Model(GPT2PreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1213,12 +1109,11 @@ class GPT2Model(GPT2PreTrainedModel): return causal_mask -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - GPT2_START_DOCSTRING, + """ ) class GPT2LMHeadModel(GPT2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1272,12 +1167,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1298,6 +1188,18 @@ class GPT2LMHeadModel(GPT2PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -1354,14 +1256,13 @@ class GPT2LMHeadModel(GPT2PreTrainedModel, GenerationMixin): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + The GPT2 Model transformer with a language modeling and a multiple-choice classification head on top e.g. for + RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the + input embeddings, the classification head takes as input the input of a specified classification token index in the + input sequence). """ -The GPT2 Model transformer with a language modeling and a multiple-choice classification head on top e.g. for -RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the -input embeddings, the classification head takes as input the input of a specified classification token index in the -input sequence). -""", - GPT2_START_DOCSTRING, ) class GPT2DoubleHeadsModel(GPT2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1419,8 +1320,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GPT2DoubleHeadsModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1441,6 +1341,18 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, GPT2DoubleHeadsModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) mc_token_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input): Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - 1]`. @@ -1452,8 +1364,6 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel, GenerationMixin): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above) - Return: - Example: ```python @@ -1549,8 +1459,8 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPT2 Model transformer with a sequence classification head on top (linear layer). [`GPT2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1561,8 +1471,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GPT2_START_DOCSTRING, + """ ) class GPT2ForSequenceClassification(GPT2PreTrainedModel): def __init__(self, config): @@ -1578,12 +1487,7 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint="microsoft/DialogRPT-updown", - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1600,6 +1504,18 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1681,13 +1597,7 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel): ) -@add_start_docstrings( - """ - GPT2 Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - GPT2_START_DOCSTRING, -) +@auto_docstring class GPT2ForTokenClassification(GPT2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1710,29 +1620,7 @@ class GPT2ForTokenClassification(GPT2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) - # fmt: off - @add_code_sample_docstrings( - checkpoint="brad1141/gpt2-finetuned-comp2", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_loss=0.25, - expected_output=[ - "Lead", - "Lead", - "Lead", - "Position", - "Lead", - "Lead", - "Lead", - "Lead", - "Lead", - "Lead", - "Lead", - "Lead", - ], - ) - # fmt: on + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1749,6 +1637,18 @@ class GPT2ForTokenClassification(GPT2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1792,13 +1692,7 @@ class GPT2ForTokenClassification(GPT2PreTrainedModel): ) -@add_start_docstrings( - """ - The GPT-2 Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - GPT2_START_DOCSTRING, -) +@auto_docstring class GPT2ForQuestionAnswering(GPT2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1813,13 +1707,7 @@ class GPT2ForQuestionAnswering(GPT2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1835,14 +1723,18 @@ class GPT2ForQuestionAnswering(GPT2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py b/src/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py index 1ba8536839..c2c21bb1c3 100644 --- a/src/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py +++ b/src/transformers/models/gpt_bigcode/modeling_gpt_bigcode.py @@ -34,9 +34,7 @@ from ...modeling_outputs import ( from ...modeling_utils import PreTrainedModel from ...pytorch_utils import is_torch_greater_or_equal_than_2_2 from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) from .configuration_gpt_bigcode import GPTBigCodeConfig @@ -48,9 +46,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "bigcode/gpt_bigcode-santacoder" -_CONFIG_FOR_DOC = "GPTBigCodeConfig" - # Fused kernels # Use separate functions for each case because conditionals prevent kernel fusion. @@ -653,12 +648,8 @@ class GPTBigCodeBlock(nn.Module): return outputs # hidden_states, present, (attentions, cross_attentions) +@auto_docstring class GPTBigCodePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GPTBigCodeConfig base_model_prefix = "transformer" supports_gradient_checkpointing = True @@ -698,95 +689,7 @@ class GPTBigCodePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GPT_BIGCODE_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GPTBigCodeConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GPT_BIGCODE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else - `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input - sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Tuple[torch.Tensor]` of length `config.n_layers`): - Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see - `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have - their past given to this model should not be passed as `input_ids` as they have already been computed. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If `past_key_values` is used, `attention_mask` needs to contain the masking strategy that was used for - `past_key_values`. In other words, the `attention_mask` always has to have the length: - `len(past_key_values) + len(input_ids)` - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.Tensor` of shape `(batch_size, input_ids_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare GPT_BIGCODE Model transformer outputting raw hidden-states without any specific head on top.", - GPT_BIGCODE_START_DOCSTRING, -) +@auto_docstring class GPTBigCodeModel(GPTBigCodePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -819,12 +722,7 @@ class GPTBigCodeModel(GPTBigCodePreTrainedModel): def set_input_embeddings(self, new_embeddings): self.wte = new_embeddings - @add_start_docstrings_to_model_forward(GPT_BIGCODE_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -841,6 +739,20 @@ class GPTBigCodeModel(GPTBigCodePreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1036,12 +948,11 @@ class GPTBigCodeModel(GPTBigCodePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPT_BIGCODE Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - GPT_BIGCODE_START_DOCSTRING, + """ ) class GPTBigCodeForCausalLM(GPTBigCodePreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1129,12 +1040,7 @@ class GPTBigCodeForCausalLM(GPTBigCodePreTrainedModel, GenerationMixin): model_kwargs["cache_position"] = torch.arange(past_length, cur_len, device=device) return model_kwargs - @add_start_docstrings_to_model_forward(GPT_BIGCODE_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1154,6 +1060,18 @@ class GPTBigCodeForCausalLM(GPTBigCodePreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.Tensor` of shape `(batch_size, input_ids_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -1214,8 +1132,8 @@ class GPTBigCodeForCausalLM(GPTBigCodePreTrainedModel, GenerationMixin): return tuple(layer_past.index_select(0, beam_idx.to(layer_past.device)) for layer_past in past_key_values) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPTBigCode Model transformer with a sequence classification head on top (linear layer). [`GPTBigCodeForSequenceClassification`] uses the last token in order to do the classification, as other causal @@ -1226,8 +1144,7 @@ class GPTBigCodeForCausalLM(GPTBigCodePreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GPT_BIGCODE_START_DOCSTRING, + """ ) class GPTBigCodeForSequenceClassification(GPTBigCodePreTrainedModel): def __init__(self, config): @@ -1239,7 +1156,7 @@ class GPTBigCodeForSequenceClassification(GPTBigCodePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT_BIGCODE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1256,6 +1173,18 @@ class GPTBigCodeForSequenceClassification(GPTBigCodePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1339,13 +1268,7 @@ class GPTBigCodeForSequenceClassification(GPTBigCodePreTrainedModel): ) -@add_start_docstrings( - """ - GPT_BIGCODE Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - GPT_BIGCODE_START_DOCSTRING, -) +@auto_docstring class GPTBigCodeForTokenClassification(GPTBigCodePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1364,7 +1287,7 @@ class GPTBigCodeForTokenClassification(GPTBigCodePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT_BIGCODE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1381,6 +1304,18 @@ class GPTBigCodeForTokenClassification(GPTBigCodePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_ids (`torch.Tensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index c343dd7f27..b67bd37c6b 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -38,9 +38,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, logging, ) @@ -64,11 +62,6 @@ _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_m logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GPTNeoConfig" - - -_CHECKPOINT_FOR_DOC = "EleutherAI/gpt-neo-1.3B" - def load_tf_weights_in_gpt_neo(model, config, gpt_neo_checkpoint_path): """Load tf checkpoints in a pytorch model""" @@ -483,12 +476,8 @@ class GPTNeoBlock(nn.Module): return outputs # hidden_states, past_kv, attentions +@auto_docstring class GPTNeoPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GPTNeoConfig load_tf_weights = load_tf_weights_in_gpt_neo base_model_prefix = "transformer" @@ -520,109 +509,7 @@ class GPTNeoPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GPT_NEO_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GPTNeoConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GPT_NEO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else - `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input - sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, input_ids_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GPT Neo Model transformer outputting raw hidden-states without any specific head on top.", - GPT_NEO_START_DOCSTRING, -) +@auto_docstring class GPTNeoModel(GPTNeoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -644,12 +531,7 @@ class GPTNeoModel(GPTNeoPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.wte = new_embeddings - @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -665,6 +547,20 @@ class GPTNeoModel(GPTNeoPreTrainedModel): return_dict: Optional[bool] = None, cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -912,12 +808,11 @@ class GPTNeoModel(GPTNeoPreTrainedModel): return causal_mask -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPT Neo Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - GPT_NEO_START_DOCSTRING, + """ ) class GPTNeoForCausalLM(GPTNeoPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -936,12 +831,7 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -960,6 +850,18 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -1031,8 +933,8 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPTNeo Model transformer with a sequence classification head on top (linear layer). [`GPTNeoForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1043,8 +945,7 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GPT_NEO_START_DOCSTRING, + """ ) class GPTNeoForSequenceClassification(GPTNeoPreTrainedModel): def __init__(self, config): @@ -1056,12 +957,7 @@ class GPTNeoForSequenceClassification(GPTNeoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1078,6 +974,18 @@ class GPTNeoForSequenceClassification(GPTNeoPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1159,13 +1067,7 @@ class GPTNeoForSequenceClassification(GPTNeoPreTrainedModel): ) -@add_start_docstrings( - """ - GPT Neo model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - GPT_NEO_START_DOCSTRING, -) +@auto_docstring class GPTNeoForTokenClassification(GPTNeoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1178,13 +1080,7 @@ class GPTNeoForTokenClassification(GPTNeoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint="EleutherAI/gpt-neo-125m", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_loss=0.25, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1201,6 +1097,18 @@ class GPTNeoForTokenClassification(GPTNeoPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1244,13 +1152,7 @@ class GPTNeoForTokenClassification(GPTNeoPreTrainedModel): ) -@add_start_docstrings( - """ - The GPT-Neo Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - GPT_NEO_START_DOCSTRING, -) +@auto_docstring class GPTNeoForQuestionAnswering(GPTNeoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1261,13 +1163,7 @@ class GPTNeoForQuestionAnswering(GPTNeoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPT_NEO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1283,14 +1179,18 @@ class GPTNeoForQuestionAnswering(GPTNeoPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/gpt_neox/modeling_gpt_neox.py b/src/transformers/models/gpt_neox/modeling_gpt_neox.py index 0cb34d04e4..00eaa2b325 100755 --- a/src/transformers/models/gpt_neox/modeling_gpt_neox.py +++ b/src/transformers/models/gpt_neox/modeling_gpt_neox.py @@ -24,16 +24,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_gpt_neox import GPTNeoXConfig @@ -46,10 +37,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "trl-internal-testing/tiny-random-GPTNeoXForCausalLM" -_CONFIG_FOR_DOC = "GPTNeoXConfig" - - class GPTNeoXMLP(nn.Module): def __init__(self, config): super().__init__() @@ -322,27 +309,7 @@ class GPTNeoXRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -GPT_NEOX_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GPTNeoXConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GPTNeoX Model outputting raw hidden-states without any specific head on top.", - GPT_NEOX_START_DOCSTRING, -) +@auto_docstring class GPTNeoXPreTrainedModel(PreTrainedModel): config_class = GPTNeoXConfig base_model_prefix = "gpt_neox" @@ -373,91 +340,8 @@ class GPTNeoXPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -_REAL_CHECKPOINT_FOR_DOC = "EleutherAI/gpt-neox-20b" - - -GPT_NEOX_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GPTNeoX Model outputting raw hidden-states without any specific head on top.", - GPT_NEOX_START_DOCSTRING, -) +@auto_docstring class GPTNeoXModel(GPTNeoXPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GPTNeoXDecoderLayer`] - - Args: - config: GPTNeoXConfig - """ - def __init__(self, config): super().__init__(config) self.config = config @@ -479,13 +363,7 @@ class GPTNeoXModel(GPTNeoXPreTrainedModel): self.embed_in = value @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -728,8 +606,10 @@ class GPTNeoXModel(GPTNeoXPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """GPTNeoX Model with a `language modeling` head on top for CLM fine-tuning.""", GPT_NEOX_START_DOCSTRING +@auto_docstring( + custom_intro=""" + GPTNeoX Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): _tied_weights_keys = ["embed_out.weight"] @@ -752,8 +632,7 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): self.embed_out = new_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -775,11 +654,6 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -830,8 +704,8 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPTNeoX Model transformer with a sequence classification head on top (linear layer). [`GPTNeoXForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -842,8 +716,7 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GPT_NEOX_START_DOCSTRING, + """ ) class GPTNeoXForSequenceClassification(GPTNeoXPreTrainedModel): def __init__(self, config): @@ -856,12 +729,7 @@ class GPTNeoXForSequenceClassification(GPTNeoXPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -941,13 +809,7 @@ class GPTNeoXForTokenClassification(GPTNeoXPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint="LarsJonasson/pythia-410m-deduped-sft-swedish", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_loss=0.25, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -997,13 +859,7 @@ class GPTNeoXForTokenClassification(GPTNeoXPreTrainedModel): ) -@add_start_docstrings( - """ - The GPT-NeoX Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - GPT_NEOX_START_DOCSTRING, -) +@auto_docstring class GPTNeoXForQuestionAnswering(GPTNeoXPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1015,13 +871,7 @@ class GPTNeoXForQuestionAnswering(GPTNeoXPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1035,17 +885,6 @@ class GPTNeoXForQuestionAnswering(GPTNeoXPreTrainedModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.gpt_neox( input_ids, attention_mask=attention_mask, diff --git a/src/transformers/models/gpt_neox/modular_gpt_neox.py b/src/transformers/models/gpt_neox/modular_gpt_neox.py index 58b697c3f1..8e01a82b03 100644 --- a/src/transformers/models/gpt_neox/modular_gpt_neox.py +++ b/src/transformers/models/gpt_neox/modular_gpt_neox.py @@ -17,31 +17,13 @@ from ...modeling_outputs import ( ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) -from ..llama.modeling_llama import ( - LlamaModel, - LlamaPreTrainedModel, - LlamaRotaryEmbedding, - rotate_half, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, logging +from ..llama.modeling_llama import LlamaModel, LlamaPreTrainedModel, LlamaRotaryEmbedding, rotate_half logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "trl-internal-testing/tiny-random-GPTNeoXForCausalLM" -_REAL_CHECKPOINT_FOR_DOC = "EleutherAI/gpt-neox-20b" -_CONFIG_FOR_DOC = "GPTNeoXConfig" - - class GPTNeoXMLP(nn.Module): def __init__(self, config): super().__init__() @@ -322,14 +304,6 @@ class GPTNeoXModel(LlamaModel, nn.Module): def set_input_embeddings(self, value): self.embed_in = value - @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -449,8 +423,10 @@ class GPTNeoXModel(LlamaModel, nn.Module): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """GPTNeoX Model with a `language modeling` head on top for CLM fine-tuning.""", GPT_NEOX_START_DOCSTRING +@auto_docstring( + custom_intro=""" + GPTNeoX Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): _tied_weights_keys = ["embed_out.weight"] @@ -473,8 +449,7 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): self.embed_out = new_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -496,11 +471,6 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -551,8 +521,8 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPTNeoX Model transformer with a sequence classification head on top (linear layer). [`GPTNeoXForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -563,8 +533,7 @@ class GPTNeoXForCausalLM(GPTNeoXPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GPT_NEOX_START_DOCSTRING, + """ ) class GPTNeoXForSequenceClassification(GPTNeoXPreTrainedModel): def __init__(self, config): @@ -577,12 +546,7 @@ class GPTNeoXForSequenceClassification(GPTNeoXPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -662,13 +626,7 @@ class GPTNeoXForTokenClassification(GPTNeoXPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint="LarsJonasson/pythia-410m-deduped-sft-swedish", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_loss=0.25, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -718,13 +676,7 @@ class GPTNeoXForTokenClassification(GPTNeoXPreTrainedModel): ) -@add_start_docstrings( - """ - The GPT-NeoX Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - GPT_NEOX_START_DOCSTRING, -) +@auto_docstring class GPTNeoXForQuestionAnswering(GPTNeoXPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -736,13 +688,7 @@ class GPTNeoXForQuestionAnswering(GPTNeoXPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(GPT_NEOX_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -756,17 +702,6 @@ class GPTNeoXForQuestionAnswering(GPTNeoXPreTrainedModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.gpt_neox( input_ids, attention_mask=attention_mask, diff --git a/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py b/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py index 3497374f76..b5c11163a6 100755 --- a/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py +++ b/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py @@ -23,16 +23,12 @@ from torch import Tensor, nn from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache -from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - is_torch_flex_attn_available, - logging, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, logging from .configuration_gpt_neox_japanese import GPTNeoXJapaneseConfig @@ -44,16 +40,9 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "abeja/gpt-neox-japanese-2.7b" -_CONFIG_FOR_DOC = "GPTNeoXJapaneseConfig" - +@auto_docstring class GPTNeoXJapanesePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GPTNeoXJapaneseConfig base_model_prefix = "gpt_neox_japanese" _no_split_modules = ["GPTNeoXJapaneseLayer"] @@ -405,90 +394,7 @@ class GPTNeoXJapaneseLayer(nn.Module): return outputs # hidden_states, present, (attentions) -GPT_NEOX_JAPANESE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~GPTNeoXJapaneseConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GPT_NEOX_JAPANESE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GPTNeoXJapanese Model transformer outputting raw hidden-states without any specific head on top.", - GPT_NEOX_JAPANESE_START_DOCSTRING, -) +@auto_docstring class GPTNeoXJapaneseModel(GPTNeoXJapanesePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -510,8 +416,7 @@ class GPTNeoXJapaneseModel(GPTNeoXJapanesePreTrainedModel): def set_input_embeddings(self, value): self.embed_in = value - @add_start_docstrings_to_model_forward(GPT_NEOX_JAPANESE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -527,8 +432,6 @@ class GPTNeoXJapaneseModel(GPTNeoXJapanesePreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple, BaseModelOutputWithPast]: r""" - Returns: - Example: ```python @@ -763,9 +666,10 @@ class GPTNeoXJapaneseModel(GPTNeoXJapanesePreTrainedModel): return causal_mask -@add_start_docstrings( - """GPTNeoXJapanese Model with a `language modeling` head on top for Classifier Model fine-tuning.""", - GPT_NEOX_JAPANESE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + GPTNeoXJapanese Model with a `language modeling` head on top for Classifier Model fine-tuning. + """ ) class GPTNeoXJapaneseForCausalLM(GPTNeoXJapanesePreTrainedModel, GenerationMixin): _tied_weights_keys = ["embed_out.weight"] @@ -786,8 +690,7 @@ class GPTNeoXJapaneseForCausalLM(GPTNeoXJapanesePreTrainedModel, GenerationMixin def set_output_embeddings(self, new_embeddings): self.embed_out = new_embeddings - @add_start_docstrings_to_model_forward(GPT_NEOX_JAPANESE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -810,8 +713,6 @@ class GPTNeoXJapaneseForCausalLM(GPTNeoXJapanesePreTrainedModel, GenerationMixin `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - Returns: - Example: ```python diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index 5dafef1d99..9e6210604e 100644 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -36,9 +36,8 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...utils import ( - add_code_sample_docstrings, add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, logging, @@ -59,10 +58,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "hf-internal-testing/tiny-random-gptj" -_REAL_CHECKPOINT_FOR_DOC = "EleutherAI/gpt-j-6B" -_CONFIG_FOR_DOC = "GPTJConfig" - def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2, dtype=torch.int64) / dim)) @@ -479,12 +474,8 @@ class GPTJBlock(nn.Module): return outputs # hidden_states, present, (attentions) +@auto_docstring class GPTJPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GPTJConfig base_model_prefix = "transformer" is_parallelizable = True @@ -517,88 +508,6 @@ class GPTJPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -GPTJ_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`GPTJConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GPTJ_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - PARALLELIZE_DOCSTRING = r""" This is an experimental feature and is a subject to change at a moment's notice. Uses a device map to distribute attention modules of the model across several devices. If no device map is given, it will evenly distribute blocks @@ -648,10 +557,7 @@ DEPARALLELIZE_DOCSTRING = r""" """ -@add_start_docstrings( - "The bare GPT-J Model transformer outputting raw hidden-states without any specific head on top.", - GPTJ_START_DOCSTRING, -) +@auto_docstring class GPTJModel(GPTJPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -721,13 +627,7 @@ class GPTJModel(GPTJPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.wte = new_embeddings - @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -743,6 +643,12 @@ class GPTJModel(GPTJPreTrainedModel): return_dict: Optional[bool] = None, cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1013,11 +919,10 @@ class GPTJModel(GPTJPreTrainedModel): return causal_mask -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPT-J Model transformer with a language modeling head on top. - """, - GPTJ_START_DOCSTRING, + """ ) class GPTJForCausalLM(GPTJPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1071,13 +976,7 @@ class GPTJForCausalLM(GPTJPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithPast, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1096,6 +995,10 @@ class GPTJForCausalLM(GPTJPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -1170,8 +1073,8 @@ class GPTJForCausalLM(GPTJPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The GPT-J Model transformer with a sequence classification head on top (linear layer). [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1182,8 +1085,7 @@ class GPTJForCausalLM(GPTJPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - GPTJ_START_DOCSTRING, + """ ) class GPTJForSequenceClassification(GPTJPreTrainedModel): def __init__(self, config): @@ -1199,13 +1101,7 @@ class GPTJForSequenceClassification(GPTJPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="ydshieh/tiny-random-gptj-for-sequence-classification", - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1222,6 +1118,10 @@ class GPTJForSequenceClassification(GPTJPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutputWithPast]: r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1304,13 +1204,7 @@ class GPTJForSequenceClassification(GPTJPreTrainedModel): ) -@add_start_docstrings( - """ - The GPT-J Model transformer with a span classification head on top for extractive question-answering tasks like - SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - GPTJ_START_DOCSTRING, -) +@auto_docstring class GPTJForQuestionAnswering(GPTJPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1325,13 +1219,7 @@ class GPTJForQuestionAnswering(GPTJPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - real_checkpoint=_REAL_CHECKPOINT_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1347,14 +1235,10 @@ class GPTJForQuestionAnswering(GPTJPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/granite/modeling_granite.py b/src/transformers/models/granite/modeling_granite.py index e909c8ae02..52eea02179 100644 --- a/src/transformers/models/granite/modeling_granite.py +++ b/src/transformers/models/granite/modeling_granite.py @@ -35,15 +35,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_granite import GraniteConfig @@ -54,7 +46,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GraniteConfig" def rotate_half(x): @@ -320,27 +311,7 @@ class GraniteDecoderLayer(GradientCheckpointingLayer): return outputs -GRANITE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GraniteConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Granite Model outputting raw hidden-states without any specific head on top.", - GRANITE_START_DOCSTRING, -) +@auto_docstring class GranitePreTrainedModel(PreTrainedModel): config_class = GraniteConfig base_model_prefix = "model" @@ -403,88 +374,8 @@ class GraniteRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -GRANITE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Granite Model outputting raw hidden-states without any specific head on top.", - GRANITE_START_DOCSTRING, -) +@auto_docstring class GraniteModel(GranitePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GraniteDecoderLayer`] - - Args: - config: GraniteConfig - """ - def __init__(self, config: GraniteConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -509,7 +400,7 @@ class GraniteModel(GranitePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GRANITE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -729,6 +620,7 @@ class GraniteModel(GranitePreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class GraniteForCausalLM(GranitePreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -762,8 +654,7 @@ class GraniteForCausalLM(GranitePreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(GRANITE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -780,19 +671,10 @@ class GraniteForCausalLM(GranitePreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 55842e7426..bfce41e6fa 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -24,24 +24,13 @@ from torch import nn from ...generation import GenerationMixin from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_peft_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_peft_available, logging from ..auto import AutoModel, AutoModelForCausalLM -from .configuration_granite_speech import ( - GraniteSpeechConfig, - GraniteSpeechEncoderConfig, -) +from .configuration_granite_speech import GraniteSpeechConfig, GraniteSpeechEncoderConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GraniteSpeechConfig" - @dataclass class GraniteSpeechCausalLMOutputWithPast(ModelOutput): @@ -295,27 +284,7 @@ class GraniteSpeechCTCEncoder(nn.Module): return hidden_states -GRANITE_SPEECH_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config (`GraniteSpeechConfig`): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Granite Speech Model outputting raw hidden-states without any specific head on top.", - GRANITE_SPEECH_START_DOCSTRING, -) +@auto_docstring class GraniteSpeechPreTrainedModel(PreTrainedModel): config_class = GraniteSpeechConfig _supports_cache_class = True @@ -341,82 +310,10 @@ class GraniteSpeechPreTrainedModel(PreTrainedModel): module.query.data.normal_() -GRANITE_SPEECH_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, audio seq len, mel feat dim)): - The tensors corresponding to the input audios. input features can be obtained using - [`AutoFeatureExtractor`]. See [`GraniteSpeechFeatureExtractor.__call__`] for details. - [`GraniteSpeechProcessor`] uses [`GraniteSpeechFeatureExtractor`] for processing audio. - input_mask (`torch.Tensor`, *optional*) - Mask for extracted audio features that should should be ignored when creating the merged - multimodal representation (i.e., due to padding). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Granite Speech model, which consists of an audio encoder, projector, and language model.""", - GRANITE_SPEECH_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Granite Speech model, which consists of an audio encoder, projector, and language model. + """ ) class GraniteSpeechForConditionalGeneration(GraniteSpeechPreTrainedModel, GenerationMixin): def __init__(self, config: GraniteSpeechConfig): @@ -461,8 +358,7 @@ class GraniteSpeechForConditionalGeneration(GraniteSpeechPreTrainedModel, Genera projected_embeds = self.projector(encoder_embeds) return projected_embeds - @add_start_docstrings_to_model_forward(GRANITE_SPEECH_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GraniteSpeechCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -482,19 +378,16 @@ class GraniteSpeechForConditionalGeneration(GraniteSpeechPreTrainedModel, Genera **lm_kwargs, ) -> Union[Tuple[torch.Tensor], GraniteSpeechCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, audio seq len, mel feat dim)): + The tensors corresponding to the input audios. input features can be obtained using + [`AutoFeatureExtractor`]. See [`GraniteSpeechFeatureExtractor.__call__`] for details. + [`GraniteSpeechProcessor`] uses [`GraniteSpeechFeatureExtractor`] for processing audio. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + input_features_mask (`torch.Tensor`, *optional*): + Mask to be applied to audio features prior to scattering into the language embeddings. """ # TODO (@alex-jw-brooks) add an example to this docstring once models are released output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions diff --git a/src/transformers/models/granitemoe/modeling_granitemoe.py b/src/transformers/models/granitemoe/modeling_granitemoe.py index 58d8093f4f..df6fd0f9aa 100644 --- a/src/transformers/models/granitemoe/modeling_granitemoe.py +++ b/src/transformers/models/granitemoe/modeling_granitemoe.py @@ -24,21 +24,11 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_layers import GradientCheckpointingLayer -from ...modeling_outputs import ( - BaseModelOutputWithPast, - MoeCausalLMOutputWithPast, - MoeModelOutputWithPast, -) +from ...modeling_outputs import BaseModelOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, logging from .configuration_granitemoe import GraniteMoeConfig @@ -50,8 +40,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GraniteMoeConfig" - # Copied from transformers.models.jetmoe.modeling_jetmoe.load_balancing_loss_func def load_balancing_loss_func( @@ -606,27 +594,7 @@ class GraniteMoeDecoderLayer(GradientCheckpointingLayer): return outputs -GRANITEMOE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GraniteMoeConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GraniteMoe Model outputting raw hidden-states without any specific head on top.", - GRANITEMOE_START_DOCSTRING, -) +@auto_docstring class GraniteMoePreTrainedModel(PreTrainedModel): config_class = GraniteMoeConfig base_model_prefix = "model" @@ -654,92 +622,8 @@ class GraniteMoePreTrainedModel(PreTrainedModel): module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) -GRANITEMOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GraniteMoe Model outputting raw hidden-states without any specific head on top.", - GRANITEMOE_START_DOCSTRING, -) +@auto_docstring class GraniteMoeModel(GraniteMoePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GraniteMoeDecoderLayer`] - - Args: - config: GraniteMoeConfig - """ - def __init__(self, config: GraniteMoeConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -771,7 +655,7 @@ class GraniteMoeModel(GraniteMoePreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(GRANITEMOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1049,8 +933,7 @@ class GraniteMoeForCausalLM(GraniteMoePreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(GRANITEMOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1069,19 +952,10 @@ class GraniteMoeForCausalLM(GraniteMoePreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, MoeCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py index 932c57ba0a..603f841421 100644 --- a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py +++ b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py @@ -35,14 +35,7 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from ...utils.import_utils import is_causal_conv1d_available, is_mamba_2_ssm_available from .configuration_granitemoehybrid import GraniteMoeHybridConfig @@ -68,9 +61,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GraniteMoeHybridConfig" - - def rotate_half(x): """Rotates half the hidden dims of the input.""" x1 = x[..., : x.shape[-1] // 2] @@ -1158,27 +1148,7 @@ class GraniteMoeHybridDecoderLayer(GradientCheckpointingLayer): return outputs -GRANITEMOEHYBRID_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GraniteMoeHybridConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GraniteMoeHybrid Model outputting raw hidden-states without any specific head on top.", - GRANITEMOEHYBRID_START_DOCSTRING, -) +@auto_docstring class GraniteMoeHybridPreTrainedModel(PreTrainedModel): config_class = GraniteMoeHybridConfig base_model_prefix = "model" @@ -1252,93 +1222,8 @@ class GraniteMoeHybridRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -GRANITEMOEHYBRID_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GraniteMoeHybrid Model outputting raw hidden-states without any specific head on top.", - GRANITEMOEHYBRID_START_DOCSTRING, -) +@auto_docstring class GraniteMoeHybridModel(GraniteMoeHybridPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. - Each layer is a [`GraniteMoeHybridDecoderLayer`] - - Args: - config: GraniteMoeHybridConfig - """ - def __init__(self, config: GraniteMoeHybridConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1371,7 +1256,7 @@ class GraniteMoeHybridModel(GraniteMoeHybridPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(GRANITEMOEHYBRID_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1741,8 +1626,7 @@ class GraniteMoeHybridForCausalLM(GraniteMoeHybridPreTrainedModel, GenerationMix def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(GRANITEMOEHYBRID_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1761,19 +1645,10 @@ class GraniteMoeHybridForCausalLM(GraniteMoeHybridPreTrainedModel, GenerationMix **kwargs, ) -> Union[Tuple, MoeCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py b/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py index c9fe70d5b0..f9dc0ec3ea 100644 --- a/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py +++ b/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py @@ -20,18 +20,9 @@ from torch import nn from ...cache_utils import Cache from ...modeling_outputs import BaseModelOutputWithPast, MoeModelOutputWithPast -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, -) +from ...utils import auto_docstring, can_return_tuple, logging from ..bamba.configuration_bamba import BambaConfig -from ..bamba.modeling_bamba import ( - BambaMixer, - BambaRMSNormGated, - HybridMambaAttentionDynamicCache, -) +from ..bamba.modeling_bamba import BambaMixer, BambaRMSNormGated, HybridMambaAttentionDynamicCache from ..granitemoeshared.modeling_granitemoeshared import ( GraniteMoeSharedAttention, GraniteMoeSharedDecoderLayer, @@ -165,27 +156,6 @@ class GraniteMoeHybridDecoderLayer(GraniteMoeSharedDecoderLayer): return outputs -GRANITEMOEHYBRID_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GraniteMoeHybridConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GraniteMoeHybrid Model outputting raw hidden-states without any specific head on top.", - GRANITEMOEHYBRID_START_DOCSTRING, -) class GraniteMoeHybridPreTrainedModel(GraniteMoeSharedPreTrainedModel): config_class = GraniteMoeHybridConfig _no_split_modules = ["GraniteMoeHybridDecoderLayer"] @@ -206,93 +176,7 @@ class GraniteMoeHybridPreTrainedModel(GraniteMoeSharedPreTrainedModel): module.weight.data.fill_(1.0) -GRANITEMOEHYBRID_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GraniteMoeHybrid Model outputting raw hidden-states without any specific head on top.", - GRANITEMOEHYBRID_START_DOCSTRING, -) class GraniteMoeHybridModel(GraniteMoeSharedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. - Each layer is a [`GraniteMoeHybridDecoderLayer`] - - Args: - config: GraniteMoeHybridConfig - """ - def __init__(self, config: GraniteMoeHybridConfig): super().__init__(config) self.layers = nn.ModuleList( @@ -300,7 +184,7 @@ class GraniteMoeHybridModel(GraniteMoeSharedModel): ) @can_return_tuple - @add_start_docstrings_to_model_forward(GRANITEMOEHYBRID_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, diff --git a/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py b/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py index 21233195e2..0845ba7b69 100644 --- a/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py +++ b/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py @@ -33,13 +33,7 @@ from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, logging from .configuration_granitemoeshared import GraniteMoeSharedConfig @@ -52,9 +46,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GraniteMoeSharedConfig" - - class GraniteMoeSharedMLP(nn.Module): """ MLP layer for shared experts @@ -517,27 +508,7 @@ class GraniteMoeSharedDecoderLayer(GradientCheckpointingLayer): return outputs -GRANITEMOESHARED_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GraniteMoeSharedConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GraniteMoeShared Model outputting raw hidden-states without any specific head on top.", - GRANITEMOESHARED_START_DOCSTRING, -) +@auto_docstring class GraniteMoeSharedPreTrainedModel(PreTrainedModel): config_class = GraniteMoeSharedConfig base_model_prefix = "model" @@ -599,92 +570,8 @@ class GraniteMoeSharedRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -GRANITEMOESHARED_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GraniteMoeShared Model outputting raw hidden-states without any specific head on top.", - GRANITEMOESHARED_START_DOCSTRING, -) +@auto_docstring class GraniteMoeSharedModel(GraniteMoeSharedPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GraniteMoeDecoderLayer`] - - Args: - config: GraniteMoeSharedConfig - """ - def __init__(self, config: GraniteMoeSharedConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -716,7 +603,7 @@ class GraniteMoeSharedModel(GraniteMoeSharedPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(GRANITEMOESHARED_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1074,8 +961,7 @@ class GraniteMoeSharedForCausalLM(GraniteMoeSharedPreTrainedModel, GenerationMix def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(GRANITEMOESHARED_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1094,19 +980,10 @@ class GraniteMoeSharedForCausalLM(GraniteMoeSharedPreTrainedModel, GenerationMix **kwargs, ) -> Union[Tuple, MoeCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/granitemoeshared/modular_granitemoeshared.py b/src/transformers/models/granitemoeshared/modular_granitemoeshared.py index 8aa2d64aa5..b89775955b 100644 --- a/src/transformers/models/granitemoeshared/modular_granitemoeshared.py +++ b/src/transformers/models/granitemoeshared/modular_granitemoeshared.py @@ -20,7 +20,7 @@ from torch import nn from ...activations import ACT2FN from ...cache_utils import Cache -from ...utils import add_start_docstrings, logging +from ...utils import logging from ..granitemoe.modeling_granitemoe import ( GraniteMoeDecoderLayer, GraniteMoeForCausalLM, @@ -33,9 +33,6 @@ from .configuration_granitemoeshared import GraniteMoeSharedConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GraniteMoeSharedConfig" - - class GraniteMoeSharedMLP(nn.Module): """ MLP layer for shared experts @@ -152,118 +149,12 @@ class GraniteMoeSharedDecoderLayer(GraniteMoeDecoderLayer): return outputs -GRANITEMOESHARED_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GraniteMoeSharedConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare GraniteMoeShared Model outputting raw hidden-states without any specific head on top.", - GRANITEMOESHARED_START_DOCSTRING, -) class GraniteMoeSharedPreTrainedModel(GraniteMoePreTrainedModel): config_class = GraniteMoeSharedConfig _no_split_modules = ["GraniteMoeSharedDecoderLayer"] -GRANITEMOESHARED_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare GraniteMoeShared Model outputting raw hidden-states without any specific head on top.", - GRANITEMOESHARED_START_DOCSTRING, -) class GraniteMoeSharedModel(GraniteMoeModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GraniteMoeDecoderLayer`] - - Args: - config: GraniteMoeSharedConfig - """ - def __init__(self, config: GraniteMoeSharedConfig): super().__init__(config) self.layers = nn.ModuleList( diff --git a/src/transformers/models/grounding_dino/image_processing_grounding_dino_fast.py b/src/transformers/models/grounding_dino/image_processing_grounding_dino_fast.py index 3d6389047c..69166a1607 100644 --- a/src/transformers/models/grounding_dino/image_processing_grounding_dino_fast.py +++ b/src/transformers/models/grounding_dino/image_processing_grounding_dino_fast.py @@ -9,8 +9,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from ...image_processing_utils import BatchFeature, get_size_dict from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, @@ -33,7 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -62,6 +60,26 @@ logger = logging.get_logger(__name__) class GroundingDinoFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): + Data format of the annotations. One of "coco_detection" or "coco_panoptic". + do_convert_annotations (`bool`, *optional*, defaults to `True`): + Controls whether to convert the annotations to the format expected by the GROUNDING_DINO model. Converts the + bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. + Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with zeros. + If `pad_size` is provided, the image will be padded to the specified dimensions. + Otherwise, the image will be padded to the maximum height and width of the batch. + pad_size (`Dict[str, int]`, *optional*): + The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size + provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest + height and width in the batch. + return_segmentation_masks (`bool`, *optional*, defaults to `False`): + Whether to return segmentation masks. + """ + format: Optional[Union[str, AnnotationFormat]] do_convert_annotations: Optional[bool] do_pad: Optional[bool] @@ -297,29 +315,7 @@ def _scale_boxes(boxes, target_sizes): return boxes -@add_start_docstrings( - "Constructs a fast GroundingDino image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the GROUNDING_DINO model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - """, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class GroundingDinoImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR @@ -593,9 +589,15 @@ class GroundingDinoImageProcessorFast(BaseImageProcessorFast): return image, pixel_mask, annotation - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ + @auto_docstring + def preprocess( + self, + images: ImageInput, + annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, + masks_path: Optional[Union[str, pathlib.Path]] = None, + **kwargs: Unpack[GroundingDinoFastImageProcessorKwargs], + ) -> BatchFeature: + r""" annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: @@ -609,32 +611,9 @@ class GroundingDinoImageProcessorFast(BaseImageProcessorFast): - "file_name" (`str`): The file name of the image. format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the GROUNDING_DINO model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. masks_path (`str` or `pathlib.Path`, *optional*): Path to the directory containing the segmentation masks. - """, - ) - def preprocess( - self, - images: ImageInput, - annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, - masks_path: Optional[Union[str, pathlib.Path]] = None, - **kwargs: Unpack[GroundingDinoFastImageProcessorKwargs], - ) -> BatchFeature: + """ if "pad_and_return_pixel_mask" in kwargs: kwargs["do_pad"] = kwargs.pop("pad_and_return_pixel_mask") logger.warning_once( diff --git a/src/transformers/models/grounding_dino/modeling_grounding_dino.py b/src/transformers/models/grounding_dino/modeling_grounding_dino.py index a741539a40..b75a5612c9 100644 --- a/src/transformers/models/grounding_dino/modeling_grounding_dino.py +++ b/src/transformers/models/grounding_dino/modeling_grounding_dino.py @@ -26,16 +26,13 @@ from torch import Tensor, nn from ...activations import ACT2FN from ...file_utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, is_timm_available, - replace_return_docstrings, requires_backends, ) from ...integrations import use_kernel_forward_from_hub from ...modeling_utils import PreTrainedModel from ...pytorch_utils import meshgrid -from ...utils import logging +from ...utils import auto_docstring, logging from ...utils.backbone_utils import load_backbone from ..auto import AutoModel from .configuration_grounding_dino import GroundingDinoConfig @@ -47,9 +44,6 @@ if is_timm_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "GroundingDinoConfig" -_CHECKPOINT_FOR_DOC = "IDEA-Research/grounding-dino-tiny" - @use_kernel_forward_from_hub("MultiScaleDeformableAttention") # Copied from transformers.models.deformable_detr.modeling_deformable_detr.MultiScaleDeformableAttention @@ -1401,6 +1395,7 @@ class GroundingDinoContrastiveEmbedding(nn.Module): return new_output +@auto_docstring class GroundingDinoPreTrainedModel(PreTrainedModel): config_class = GroundingDinoConfig base_model_prefix = "model" @@ -1476,75 +1471,6 @@ class GroundingDinoPreTrainedModel(PreTrainedModel): module.gradient_checkpointing = value -GROUNDING_DINO_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`GroundingDinoConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GROUNDING_DINO_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`AutoImageProcessor`]. See [`GroundingDinoImageProcessor.__call__`] for - details. - - input_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`BertTokenizer.__call__`] for details. - - token_type_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: 0 corresponds to a `sentence A` token, 1 corresponds to a `sentence B` token - - [What are token type IDs?](../glossary#token-type-ids) - - attention_mask (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are real (i.e. **not masked**), - - 0 for tokens that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state_vision`, *optional*: `last_hidden_state_text`, *optional*: - `vision_hidden_states`, *optional*: `text_hidden_states`, *optional*: `attentions`) - `last_hidden_state_vision` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence - of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the - decoder. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - class GroundingDinoEncoder(GroundingDinoPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* deformable attention layers. Each layer is a @@ -1999,12 +1925,11 @@ def generate_masks_with_special_tokens_and_transfer_map(input_ids: torch.LongTen return attention_mask, position_ids.to(torch.long) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The bare Grounding DINO Model (consisting of a backbone and encoder-decoder Transformer) outputting raw hidden-states without any specific head on top. - """, - GROUNDING_DINO_START_DOCSTRING, + """ ) class GroundingDinoModel(GroundingDinoPreTrainedModel): def __init__(self, config: GroundingDinoConfig): @@ -2154,8 +2079,7 @@ class GroundingDinoModel(GroundingDinoPreTrainedModel): object_query = self.enc_output_norm(self.enc_output(object_query)) return object_query, output_proposals - @add_start_docstrings_to_model_forward(GROUNDING_DINO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GroundingDinoModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -2169,7 +2093,16 @@ class GroundingDinoModel(GroundingDinoPreTrainedModel): return_dict=None, ): r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`BertTokenizer.__call__`] for details. + token_type_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: 0 corresponds to a `sentence A` token, 1 corresponds to a `sentence B` token + + [What are token type IDs?](../glossary#token-type-ids) Examples: @@ -2503,12 +2436,11 @@ def build_text_mask(logits, attention_mask): return text_mask.bool() -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Grounding DINO Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on top, for tasks such as COCO detection. - """, - GROUNDING_DINO_START_DOCSTRING, + """ ) class GroundingDinoForObjectDetection(GroundingDinoPreTrainedModel): # When using clones, all layers > 0 will be clones, but layer 0 *is* required @@ -2541,8 +2473,7 @@ class GroundingDinoForObjectDetection(GroundingDinoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GROUNDING_DINO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GroundingDinoObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -2557,14 +2488,22 @@ class GroundingDinoForObjectDetection(GroundingDinoPreTrainedModel): labels: Optional[List[Dict[str, Union[torch.LongTensor, torch.FloatTensor]]]] = None, ): r""" + input_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`BertTokenizer.__call__`] for details. + token_type_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: 0 corresponds to a `sentence A` token, 1 corresponds to a `sentence B` token + + [What are token type IDs?](../glossary#token-type-ids) labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/groupvit/modeling_groupvit.py b/src/transformers/models/groupvit/modeling_groupvit.py index 6a5c235cf8..69628249b3 100644 --- a/src/transformers/models/groupvit/modeling_groupvit.py +++ b/src/transformers/models/groupvit/modeling_groupvit.py @@ -27,21 +27,12 @@ from ...activations import ACT2FN from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_groupvit import GroupViTConfig, GroupViTTextConfig, GroupViTVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "nvidia/groupvit-gcc-yfcc" - # contrastive loss function, adapted from # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/CLIP.html @@ -751,12 +742,8 @@ class GroupViTEncoderLayer(nn.Module): return outputs +@auto_docstring class GroupViTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = GroupViTConfig base_model_prefix = "groupvit" supports_gradient_checkpointing = True @@ -795,102 +782,6 @@ class GroupViTPreTrainedModel(PreTrainedModel): nn.init.normal_(module.fc2.weight, std=in_proj_std) -GROUPVIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`GroupViTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -GROUPVIT_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -GROUPVIT_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -GROUPVIT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class GroupViTVisionEncoder(nn.Module): def __init__(self, config: GroupViTVisionConfig) -> None: super().__init__() @@ -1058,8 +949,7 @@ class GroupViTTextTransformer(nn.Module): # For `pooled_output` computation self.eos_token_id = config.eos_token_id - @add_start_docstrings_to_model_forward(GROUPVIT_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=GroupViTTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1069,10 +959,6 @@ class GroupViTTextTransformer(nn.Module): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1158,8 +1044,7 @@ class GroupViTTextModel(GroupViTPreTrainedModel): def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value - @add_start_docstrings_to_model_forward(GROUPVIT_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=GroupViTTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1170,8 +1055,6 @@ class GroupViTTextModel(GroupViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -1206,8 +1089,7 @@ class GroupViTVisionTransformer(nn.Module): self.encoder = GroupViTVisionEncoder(config) self.layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(GROUPVIT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=GroupViTVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1215,10 +1097,6 @@ class GroupViTVisionTransformer(nn.Module): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1267,8 +1145,7 @@ class GroupViTVisionModel(GroupViTPreTrainedModel): def get_input_embeddings(self) -> GroupViTPatchEmbeddings: return self.vision_model.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(GROUPVIT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=GroupViTVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1277,8 +1154,6 @@ class GroupViTVisionModel(GroupViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -1306,7 +1181,7 @@ class GroupViTVisionModel(GroupViTPreTrainedModel): ) -@add_start_docstrings(GROUPVIT_START_DOCSTRING) +@auto_docstring class GroupViTModel(GroupViTPreTrainedModel): config_class = GroupViTConfig @@ -1353,7 +1228,7 @@ class GroupViTModel(GroupViTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(GROUPVIT_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1400,7 +1275,7 @@ class GroupViTModel(GroupViTPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(GROUPVIT_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1449,8 +1324,7 @@ class GroupViTModel(GroupViTPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(GROUPVIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=GroupViTModelOutput, config_class=GroupViTConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1464,7 +1338,10 @@ class GroupViTModel(GroupViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, GroupViTModelOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. + output_segmentation (`bool`, *optional*): + Whether or not to return the segmentation logits. Examples: diff --git a/src/transformers/models/helium/modeling_helium.py b/src/transformers/models/helium/modeling_helium.py index 7d3e6cca23..5c83db0ab7 100644 --- a/src/transformers/models/helium/modeling_helium.py +++ b/src/transformers/models/helium/modeling_helium.py @@ -40,16 +40,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_helium import HeliumConfig @@ -61,9 +52,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/helium-7b" -_CONFIG_FOR_DOC = "HeliumConfig" - class HeliumRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -338,27 +326,7 @@ class HeliumDecoderLayer(GradientCheckpointingLayer): return outputs -HELIUM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`HeliumConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Helium Model outputting raw hidden-states without any specific head on top.", - HELIUM_START_DOCSTRING, -) +@auto_docstring class HeliumPreTrainedModel(PreTrainedModel): config_class = HeliumConfig base_model_prefix = "model" @@ -387,88 +355,8 @@ class HeliumPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -HELIUM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Helium Model outputting raw hidden-states without any specific head on top.", - HELIUM_START_DOCSTRING, -) +@auto_docstring class HeliumModel(HeliumPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`HeliumDecoderLayer`] - - Args: - config: HeliumConfig - """ - def __init__(self, config: HeliumConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -492,7 +380,7 @@ class HeliumModel(HeliumPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(HELIUM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -714,6 +602,7 @@ class HeliumModel(HeliumPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class HeliumForCausalLM(HeliumPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -747,8 +636,7 @@ class HeliumForCausalLM(HeliumPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(HELIUM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -765,19 +653,10 @@ class HeliumForCausalLM(HeliumPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -832,8 +711,8 @@ class HeliumForCausalLM(HeliumPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Helium Model transformer with a sequence classification head on top (linear layer). [`HeliumForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -844,8 +723,7 @@ class HeliumForCausalLM(HeliumPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - HELIUM_START_DOCSTRING, + """ ) class HeliumForSequenceClassification(HeliumPreTrainedModel): def __init__(self, config: HeliumConfig): @@ -864,7 +742,7 @@ class HeliumForSequenceClassification(HeliumPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(HELIUM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -933,13 +811,7 @@ class HeliumForSequenceClassification(HeliumPreTrainedModel): ) -@add_start_docstrings( - """ - The Helium Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - HELIUM_START_DOCSTRING, -) +@auto_docstring class HeliumForTokenClassification(HeliumPreTrainedModel): def __init__(self, config: HeliumConfig): super().__init__(config) @@ -964,12 +836,7 @@ class HeliumForTokenClassification(HeliumPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(HELIUM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/helium/modular_helium.py b/src/transformers/models/helium/modular_helium.py index 0c9be5ec80..ed26ff852f 100644 --- a/src/transformers/models/helium/modular_helium.py +++ b/src/transformers/models/helium/modular_helium.py @@ -21,21 +21,9 @@ import torch.nn as nn import torch.utils.checkpoint from ...utils import logging -from ..gemma.modeling_gemma import ( - GemmaForCausalLM, - GemmaForSequenceClassification, - GemmaForTokenClassification, -) -from ..granite.modeling_granite import ( - GraniteAttention, -) -from ..llama.modeling_llama import ( - LlamaDecoderLayer, - LlamaMLP, - LlamaModel, - LlamaPreTrainedModel, - LlamaRotaryEmbedding, -) +from ..gemma.modeling_gemma import GemmaForCausalLM, GemmaForSequenceClassification, GemmaForTokenClassification +from ..granite.modeling_granite import GraniteAttention +from ..llama.modeling_llama import LlamaDecoderLayer, LlamaMLP, LlamaModel, LlamaPreTrainedModel, LlamaRotaryEmbedding from .configuration_helium import HeliumConfig diff --git a/src/transformers/models/hgnet_v2/modeling_hgnet_v2.py b/src/transformers/models/hgnet_v2/modeling_hgnet_v2.py index d4cc2e929b..4411a50488 100644 --- a/src/transformers/models/hgnet_v2/modeling_hgnet_v2.py +++ b/src/transformers/models/hgnet_v2/modeling_hgnet_v2.py @@ -30,21 +30,16 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput, BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings +from ...utils import auto_docstring from ...utils.backbone_utils import BackboneMixin from .configuration_hgnet_v2 import HGNetV2Config # General docstring -_CONFIG_FOR_DOC = "HGNetV2Config" +@auto_docstring class HGNetV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = HGNetV2Config base_model_prefix = "hgnetv2" main_input_name = "pixel_values" @@ -346,20 +341,6 @@ class HGNetV2Encoder(nn.Module): ) -HGNet_V2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`RTDetrImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class HGNetV2Backbone(HGNetV2PreTrainedModel, BackboneMixin): def __init__(self, config: HGNetV2Config): super().__init__(config) @@ -372,14 +353,11 @@ class HGNetV2Backbone(HGNetV2PreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(HGNet_V2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python @@ -427,24 +405,11 @@ class HGNetV2Backbone(HGNetV2PreTrainedModel, BackboneMixin): ) -HGNet_V2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`HGNetV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" HGNetV2 Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - HGNet_V2_START_DOCSTRING, + """ ) class HGNetV2ForImageClassification(HGNetV2PreTrainedModel): def __init__(self, config: HGNetV2Config): @@ -462,8 +427,7 @@ class HGNetV2ForImageClassification(HGNetV2PreTrainedModel): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(HGNet_V2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -477,8 +441,6 @@ class HGNetV2ForImageClassification(HGNetV2PreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python >>> import torch diff --git a/src/transformers/models/hgnet_v2/modular_hgnet_v2.py b/src/transformers/models/hgnet_v2/modular_hgnet_v2.py index 478aabbf7a..2b4c73a1a5 100644 --- a/src/transformers/models/hgnet_v2/modular_hgnet_v2.py +++ b/src/transformers/models/hgnet_v2/modular_hgnet_v2.py @@ -28,7 +28,9 @@ from ...modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings +from ...utils import ( + auto_docstring, +) from ...utils.backbone_utils import BackboneConfigMixin, BackboneMixin, get_aligned_output_features_output_indices from ..rt_detr.modeling_rt_detr_resnet import RTDetrResNetConvLayer @@ -159,15 +161,10 @@ class HGNetV2Config(BackboneConfigMixin, PretrainedConfig): # General docstring -_CONFIG_FOR_DOC = "HGNetV2Config" +@auto_docstring class HGNetV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = HGNetV2Config base_model_prefix = "hgnetv2" main_input_name = "pixel_values" @@ -467,31 +464,6 @@ class HGNetV2Encoder(nn.Module): ) -HGNet_V2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`HGNetV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -HGNet_V2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`RTDetrImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class HGNetV2Backbone(HGNetV2PreTrainedModel, BackboneMixin): def __init__(self, config: HGNetV2Config): super().__init__(config) @@ -504,14 +476,11 @@ class HGNetV2Backbone(HGNetV2PreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(HGNet_V2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python @@ -559,12 +528,11 @@ class HGNetV2Backbone(HGNetV2PreTrainedModel, BackboneMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" HGNetV2 Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - HGNet_V2_START_DOCSTRING, + """ ) class HGNetV2ForImageClassification(HGNetV2PreTrainedModel): def __init__(self, config: HGNetV2Config): @@ -582,8 +550,7 @@ class HGNetV2ForImageClassification(HGNetV2PreTrainedModel): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(HGNet_V2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -597,8 +564,6 @@ class HGNetV2ForImageClassification(HGNetV2PreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python >>> import torch diff --git a/src/transformers/models/hiera/modeling_hiera.py b/src/transformers/models/hiera/modeling_hiera.py index fa01dd909d..613d8796c8 100644 --- a/src/transformers/models/hiera/modeling_hiera.py +++ b/src/transformers/models/hiera/modeling_hiera.py @@ -32,31 +32,13 @@ from ...modeling_outputs import ( ModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from ...utils.backbone_utils import BackboneMixin from .configuration_hiera import HieraConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "HieraConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/hiera-tiny-224-hf" -_EXPECTED_OUTPUT_SHAPE = [1, 49, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/hiera-tiny-224-in1k-hf" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - @dataclass class HieraEncoderOutput(ModelOutput): @@ -837,12 +819,8 @@ def unroll( return hidden_states +@auto_docstring class HieraPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = HieraConfig base_model_prefix = "hiera" main_input_name = "pixel_values" @@ -869,42 +847,6 @@ class HieraPreTrainedModel(PreTrainedModel): nn.init.constant_(module.weight, self.config.layer_norm_init) -HIERA_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`HieraConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -HIERA_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`BitImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class HieraPooler(nn.Module): def __init__(self, config: HieraConfig): super().__init__() @@ -920,18 +862,15 @@ class HieraPooler(nn.Module): return pooled_output -@add_start_docstrings( - "The bare Hiera Model transformer outputting raw hidden-states without any specific head on top.", - HIERA_START_DOCSTRING, - """ - add_pooling_layer (`bool`, *optional*, defaults to `True`): - Whether or not to apply pooling layer. - is_mae (`bool`, *optional*, defaults to `False`): - Whether or not to run the model on MAE mode. - """, -) +@auto_docstring class HieraModel(HieraPreTrainedModel): def __init__(self, config: HieraConfig, add_pooling_layer: bool = True, is_mae: bool = False): + r""" + add_pooling_layer (`bool`, *optional*, defaults to `True`): + Whether or not to apply pooling layer. + is_mae (`bool`, *optional*, defaults to `False`): + Whether or not to run the model on MAE mode. + """ super().__init__(config) self.num_features = int(config.embed_dim * config.embed_dim_multiplier ** (len(config.depths) - 1)) @@ -956,14 +895,7 @@ class HieraModel(HieraPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(HIERA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=HieraModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -975,9 +907,8 @@ class HieraModel(HieraPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*) which is - mainly used for testing purposes to control randomness and maintain the reproducibility - when is_mae is set to True. + noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*): + Mainly used for testing purposes to control randomness and maintain the reproducibility """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1208,8 +1139,9 @@ class HieraMultiScaleHead(nn.Module): return hidden_states -@add_start_docstrings( - """The Hiera Model transformer with the decoder on top for self-supervised pre-training. +@auto_docstring( + custom_intro=""" + The Hiera Model transformer with the decoder on top for self-supervised pre-training. @@ -1217,8 +1149,7 @@ class HieraMultiScaleHead(nn.Module): directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - """, - HIERA_START_DOCSTRING, + """ ) class HieraForPreTraining(HieraPreTrainedModel): def __init__(self, config: HieraConfig) -> None: @@ -1261,8 +1192,7 @@ class HieraForPreTraining(HieraPreTrainedModel): return loss - @add_start_docstrings_to_model_forward(HIERA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=HieraForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1274,11 +1204,8 @@ class HieraForPreTraining(HieraPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[tuple, HieraForPreTrainingOutput]: r""" - noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*) which is - mainly used for testing purposes to control randomness and maintain the reproducibility - when is_mae is set to True. - - Returns: + noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*): + Mainly used for testing purposes to control randomness and maintain the reproducibility Examples: ```python @@ -1356,8 +1283,8 @@ class HieraForPreTraining(HieraPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Hiera Model transformer with an image classification head on top (a linear layer on top of the final hidden state with average pooling) e.g. for ImageNet. @@ -1368,8 +1295,7 @@ class HieraForPreTraining(HieraPreTrainedModel): position embeddings to the higher resolution. - """, - HIERA_START_DOCSTRING, + """ ) class HieraForImageClassification(HieraPreTrainedModel): def __init__(self, config: HieraConfig) -> None: @@ -1386,13 +1312,7 @@ class HieraForImageClassification(HieraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(HIERA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=HieraForImageClassificationOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values, @@ -1466,11 +1386,10 @@ class HieraForImageClassification(HieraPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Hiera backbone, to be used with frameworks like DETR and MaskFormer. - """, - HIERA_START_DOCSTRING, + """ ) class HieraBackbone(HieraPreTrainedModel, BackboneMixin): def __init__(self, config: HieraConfig): diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py index 143ec53d54..575239e4d8 100755 --- a/src/transformers/models/hubert/modeling_hubert.py +++ b/src/transformers/models/hubert/modeling_hubert.py @@ -18,13 +18,7 @@ from ...integrations.fsdp import is_fsdp_managed_module from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_hubert import HubertConfig @@ -34,12 +28,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/hubert-large-ls960-ft" - -# General docstring -_CONFIG_FOR_DOC = "HubertConfig" - class HubertPositionalConvEmbedding(nn.Module): def __init__(self, config): @@ -936,12 +924,8 @@ class HubertEncoderStableLayerNorm(nn.Module): ) +@auto_docstring class HubertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = HubertConfig base_model_prefix = "hubert" main_input_name = "input_values" @@ -1129,67 +1113,7 @@ def _compute_mask_indices( return spec_aug_mask -HUBERT_START_DOCSTRING = r""" - Hubert was proposed in [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden - Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, - Ruslan Salakhutdinov, Abdelrahman Mohamed. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`HubertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -HUBERT_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, such as - [hubert-base](https://huggingface.co/facebook/hubert-base-ls960), `attention_mask` should **not** be passed - to avoid degraded performance when doing batched inference. For such models `input_values` should simply be - padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly different - results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Hubert Model transformer outputting raw hidden-states without any specific head on top.", - HUBERT_START_DOCSTRING, -) +@auto_docstring class HubertModel(HubertPreTrainedModel): def __init__(self, config: HubertConfig): super().__init__(config) @@ -1255,8 +1179,7 @@ class HubertModel(HubertPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1266,9 +1189,10 @@ class HubertModel(HubertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: - """ - - Returns: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. Example: @@ -1332,16 +1256,19 @@ class HubertModel(HubertPreTrainedModel): _HIDDEN_STATES_START_POSITION = 1 -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 22.68 - - -@add_start_docstrings( - """Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - HUBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) class HubertForCTC(HubertPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`HubertForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.hubert = HubertModel(config) @@ -1412,14 +1339,7 @@ class HubertForCTC(HubertPreTrainedModel): for param in self.hubert.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1491,17 +1411,11 @@ class HubertForCTC(HubertPreTrainedModel): ) -_SEQ_CLASS_CHECKPOINT = "superb/hubert-base-superb-ks" -_SEQ_CLASS_EXPECTED_OUTPUT = "'_unknown_'" -_SEQ_CLASS_EXPECTED_LOSS = 8.53 - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - HUBERT_START_DOCSTRING, + """ ) class HubertForSequenceClassification(HubertPreTrainedModel): def __init__(self, config): @@ -1548,15 +1462,7 @@ class HubertForSequenceClassification(HubertPreTrainedModel): for param in self.hubert.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_SEQ_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1567,6 +1473,11 @@ class HubertForSequenceClassification(HubertPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`HubertProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/hubert/modular_hubert.py b/src/transformers/models/hubert/modular_hubert.py index b5138fdfb5..b3e3d24cc0 100644 --- a/src/transformers/models/hubert/modular_hubert.py +++ b/src/transformers/models/hubert/modular_hubert.py @@ -5,14 +5,9 @@ import torch.nn as nn from ...activations import ACT2FN from ...integrations.deepspeed import is_deepspeed_zero3_enabled -from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput +from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) +from ...utils import auto_docstring from ..wav2vec2.modeling_wav2vec2 import ( Wav2Vec2Encoder, Wav2Vec2EncoderStableLayerNorm, @@ -27,22 +22,6 @@ from .configuration_hubert import HubertConfig _HIDDEN_STATES_START_POSITION = 1 -# General docstring -_CONFIG_FOR_DOC = "HubertConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/hubert-large-ls960-ft" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - - -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 22.68 - - -_SEQ_CLASS_CHECKPOINT = "superb/hubert-base-superb-ks" -_SEQ_CLASS_EXPECTED_OUTPUT = "'_unknown_'" -_SEQ_CLASS_EXPECTED_LOSS = 8.53 - class HubertPositionalConvEmbedding(nn.Module): def __init__(self, config): @@ -128,12 +107,8 @@ class HubertEncoderStableLayerNorm(Wav2Vec2EncoderStableLayerNorm): pass +@auto_docstring class HubertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = HubertConfig base_model_prefix = "hubert" main_input_name = "input_values" @@ -202,67 +177,6 @@ class HubertPreTrainedModel(PreTrainedModel): return attention_mask -HUBERT_START_DOCSTRING = r""" - Hubert was proposed in [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden - Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, - Ruslan Salakhutdinov, Abdelrahman Mohamed. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`HubertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -HUBERT_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, such as - [hubert-base](https://huggingface.co/facebook/hubert-base-ls960), `attention_mask` should **not** be passed - to avoid degraded performance when doing batched inference. For such models `input_values` should simply be - padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly different - results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Hubert Model transformer outputting raw hidden-states without any specific head on top.", - HUBERT_START_DOCSTRING, -) class HubertModel(Wav2Vec2Model, HubertPreTrainedModel): def __init__(self, config: HubertConfig): super().__init__(config) @@ -289,8 +203,6 @@ class HubertModel(Wav2Vec2Model, HubertPreTrainedModel): def freeze_feature_encoder(self): raise AttributeError("Not needed for Hubert") - @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) def forward( self, input_values: Optional[torch.Tensor], @@ -300,9 +212,10 @@ class HubertModel(Wav2Vec2Model, HubertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: - """ - - Returns: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. Example: @@ -363,46 +276,12 @@ class HubertModel(Wav2Vec2Model, HubertPreTrainedModel): ) -@add_start_docstrings( - """Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - HUBERT_START_DOCSTRING, -) class HubertForCTC(Wav2Vec2ForCTC): pass - @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - -@add_start_docstrings( - """ - Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like - SUPERB Keyword Spotting. - """, - HUBERT_START_DOCSTRING, -) class HubertForSequenceClassification(Wav2Vec2ForSequenceClassification): pass - @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_SEQ_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - __all__ = ["HubertForCTC", "HubertForSequenceClassification", "HubertModel", "HubertPreTrainedModel"] diff --git a/src/transformers/models/ibert/modeling_ibert.py b/src/transformers/models/ibert/modeling_ibert.py index 1a922b1e29..9d322e5eca 100644 --- a/src/transformers/models/ibert/modeling_ibert.py +++ b/src/transformers/models/ibert/modeling_ibert.py @@ -37,16 +37,13 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_ibert import IBertConfig from .quant_modules import IntGELU, IntLayerNorm, IntSoftmax, QuantAct, QuantEmbedding, QuantLinear logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "kssteven/ibert-roberta-base" -_CONFIG_FOR_DOC = "IBertConfig" - class IBertEmbeddings(nn.Module): """ @@ -627,12 +624,8 @@ class IBertPooler(nn.Module): return pooled_output +@auto_docstring class IBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = IBertConfig base_model_prefix = "ibert" @@ -658,76 +651,7 @@ class IBertPreTrainedModel(PreTrainedModel): raise NotImplementedError("`resize_token_embeddings` is not supported for I-BERT.") -IBERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`IBertConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -IBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare I-BERT Model transformer outputting raw hidden-states without any specific head on top.", - IBERT_START_DOCSTRING, -) +@auto_docstring class IBertModel(IBertPreTrainedModel): """ @@ -739,6 +663,10 @@ class IBertModel(IBertPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config self.quant_mode = config.quant_mode @@ -765,12 +693,7 @@ class IBertModel(IBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -849,7 +772,7 @@ class IBertModel(IBertPreTrainedModel): ) -@add_start_docstrings("""I-BERT Model with a `language modeling` head on top.""", IBERT_START_DOCSTRING) +@auto_docstring class IBertForMaskedLM(IBertPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.bias", "lm_head.decoder.weight"] @@ -869,13 +792,7 @@ class IBertForMaskedLM(IBertPreTrainedModel): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -894,8 +811,6 @@ class IBertForMaskedLM(IBertPreTrainedModel): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -961,12 +876,11 @@ class IBertLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - IBERT_START_DOCSTRING, + """ ) class IBertForSequenceClassification(IBertPreTrainedModel): def __init__(self, config): @@ -979,12 +893,7 @@ class IBertForSequenceClassification(IBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1054,13 +963,7 @@ class IBertForSequenceClassification(IBertPreTrainedModel): ) -@add_start_docstrings( - """ - I-BERT Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - IBERT_START_DOCSTRING, -) +@auto_docstring class IBertForMultipleChoice(IBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1072,12 +975,7 @@ class IBertForMultipleChoice(IBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1092,10 +990,34 @@ class IBertForMultipleChoice(IBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[MultipleChoiceModelOutput, Tuple[torch.FloatTensor]]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1144,13 +1066,7 @@ class IBertForMultipleChoice(IBertPreTrainedModel): ) -@add_start_docstrings( - """ - I-BERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - IBERT_START_DOCSTRING, -) +@auto_docstring class IBertForTokenClassification(IBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1163,12 +1079,7 @@ class IBertForTokenClassification(IBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1241,13 +1152,7 @@ class IBertClassificationHead(nn.Module): return hidden_states -@add_start_docstrings( - """ - I-BERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - IBERT_START_DOCSTRING, -) +@auto_docstring class IBertForQuestionAnswering(IBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1259,12 +1164,7 @@ class IBertForQuestionAnswering(IBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(IBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1279,16 +1179,6 @@ class IBertForQuestionAnswering(IBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[QuestionAnsweringModelOutput, Tuple[torch.FloatTensor]]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.ibert( diff --git a/src/transformers/models/idefics/modeling_idefics.py b/src/transformers/models/idefics/modeling_idefics.py index ebe43db459..80c9af0f91 100644 --- a/src/transformers/models/idefics/modeling_idefics.py +++ b/src/transformers/models/idefics/modeling_idefics.py @@ -36,15 +36,7 @@ from ...modeling_outputs import ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PretrainedConfig, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_idefics import IdeficsConfig from .perceiver import IdeficsPerceiverResampler from .vision import IdeficsVisionEmbeddings, IdeficsVisionTransformer @@ -58,8 +50,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "IdeficsConfig" - @dataclass class IdeficsBaseModelOutputWithPast(ModelOutput): @@ -919,27 +909,7 @@ class IdeficsGatedCrossAttentionLayer(nn.Module): return outputs -LLAMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`IdeficsConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAMA_START_DOCSTRING, -) +@auto_docstring class IdeficsPreTrainedModel(PreTrainedModel): config_class = IdeficsConfig base_model_prefix = "model" @@ -985,79 +955,10 @@ class IdeficsPreTrainedModel(PreTrainedModel): module.latents.data.normal_() -LLAMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAMA_START_DOCSTRING, -) +@auto_docstring class IdeficsModel(IdeficsPreTrainedModel): """ Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`] @@ -1138,7 +1039,7 @@ class IdeficsModel(IdeficsPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1158,6 +1059,14 @@ class IdeficsModel(IdeficsPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, IdeficsBaseModelOutputWithPast]: + r""" + image_encoder_embeddings (`torch.FloatTensor`, *optional*): + The output of the image encoder. + perceiver_embeddings (`torch.FloatTensor`, *optional*): + The output of the perceiver resampler. + image_attention_mask (`torch.LongTensor`, *optional*): + The attention mask for the image encoder. + """ device = input_ids.device if input_ids is not None else inputs_embeds.device output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -1591,8 +1500,7 @@ class IdeficsForVisionText2Text(IdeficsPreTrainedModel, GenerationMixin): output_embeddings.out_additional_features = input_embeddings.num_additional_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=IdeficsCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1614,12 +1522,16 @@ class IdeficsForVisionText2Text(IdeficsPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, IdeficsCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + image_encoder_embeddings (`torch.FloatTensor`, *optional*): + The output of the image encoder. + perceiver_embeddings (`torch.FloatTensor`, *optional*): + The output of the perceiver resampler. + image_attention_mask (`torch.LongTensor`, *optional*): + The attention mask for the image encoder. Example: diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index e5e3b9f3a5..a07e480d5c 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -29,22 +29,13 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, logging from ..auto import AutoModel from .configuration_idefics2 import Idefics2Config, Idefics2PerceiverConfig, Idefics2VisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Idefics2Config" - @dataclass class Idefics2BaseModelOutputWithPast(ModelOutput): @@ -486,27 +477,7 @@ class Idefics2Encoder(nn.Module): ) -IDEFICS2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Idefics2Config`] or [`Idefics2VisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Idefics2 Model outputting raw hidden-states without any specific head on top.", - IDEFICS2_START_DOCSTRING, -) +@auto_docstring class Idefics2PreTrainedModel(PreTrainedModel): config_class = Idefics2Config base_model_prefix = "model" @@ -543,28 +514,10 @@ class Idefics2PreTrainedModel(PreTrainedModel): module.latents.data.fill_(1.0) -IDEFICS2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`CLIPImageProcessor`] for processing images). - pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): - Mask to avoid performing attention on padding pixel indices. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """Idefics2 vision encoder model that returnss raw image embeddings.""", - IDEFICS2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Idefics2 vision encoder model that returnss raw image embeddings. + """ ) class Idefics2VisionTransformer(Idefics2PreTrainedModel): config_class = Idefics2VisionConfig @@ -588,6 +541,7 @@ class Idefics2VisionTransformer(Idefics2PreTrainedModel): def set_input_embeddings(self, value): self.embeddings = value + @auto_docstring def forward( self, pixel_values, @@ -596,6 +550,10 @@ class Idefics2VisionTransformer(Idefics2PreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: + r""" + patch_attention_mask (`torch.BoolTensor` of shape `(batch_size, num_patches_height, num_patches_width)`, *optional*): + The attention mask for the patches. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -843,25 +801,10 @@ class Idefics2PerceiverLayer(nn.Module): return outputs -IDEFICS2_INPUTS_DOCSTRING = r""" - Args: - context (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`): - The hidden states of the image after vision encoder and modality projection. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) -""" - - -@add_start_docstrings( - "Idefics2 perceiver resampler model that performs `depth` blocks of cross-attention with a fixed ", - "`n_latents` inputs to decrease embedding sequence length. The Resampler acts as a form of learned pooling and ", - "is derived from [Perceiver: General Perception with Iterative Attention](https://arxiv.org/abs/2103.03206)", - IDEFICS2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Idefics2 perceiver resampler model that performs `depth` blocks of cross-attention with a fixed + """ ) class Idefics2PerceiverResampler(Idefics2PreTrainedModel): config_class = Idefics2PerceiverConfig @@ -886,11 +829,16 @@ class Idefics2PerceiverResampler(Idefics2PreTrainedModel): self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" + @auto_docstring def forward( self, context: torch.Tensor, attention_mask: torch.Tensor, ) -> torch.Tensor: + r""" + context (`torch.FloatTensor` of shape `(batch, seq_len, embed_dim)`): + Input to the layer. + """ # seq embed -> bsz seq embed latents = self.latents.unsqueeze(0).expand((context.shape[0], *self.latents.size())) @@ -940,83 +888,10 @@ class Idefics2Connector(nn.Module): return image_hidden_states -IDEFICS2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`CLIPImageProcessor`] for processing images). - pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): - Mask to avoid performing attention on padding pixel indices. - image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The hidden states of the image encoder after modality projection and perceiver resampling. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """Idefics2 model consisting of a SIGLIP vision encoder and Mistral language decoder""", - IDEFICS2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Idefics2 model consisting of a SIGLIP vision encoder and Mistral language decoder + """ ) class Idefics2Model(Idefics2PreTrainedModel): def __init__(self, config: Idefics2Config): @@ -1094,8 +969,8 @@ class Idefics2Model(Idefics2PreTrainedModel): return new_inputs_embeds @can_return_tuple - @add_start_docstrings_to_model_forward( - """ + @auto_docstring( + custom_intro=""" Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where max_num_images is the maximum number of images among the batch_size samples in the batch. @@ -1104,8 +979,7 @@ class Idefics2Model(Idefics2PreTrainedModel): For efficiency, we only pass through the vision_model's forward the real images by discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3. - """, - IDEFICS2_INPUTS_DOCSTRING, + """ ) def forward( self, @@ -1124,6 +998,12 @@ class Idefics2Model(Idefics2PreTrainedModel): return_dict: Optional[bool] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, Idefics2BaseModelOutputWithPast]: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The hidden states of the image encoder after modality projection and perceiver resampling. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1251,9 +1131,10 @@ class Idefics2Model(Idefics2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The Idefics2 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. """, - IDEFICS2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Idefics2 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. + """ ) class Idefics2ForConditionalGeneration(Idefics2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1300,8 +1181,7 @@ class Idefics2ForConditionalGeneration(Idefics2PreTrainedModel, GenerationMixin) self.lm_head = new_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(IDEFICS2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Idefics2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1322,20 +1202,15 @@ class Idefics2ForConditionalGeneration(Idefics2PreTrainedModel, GenerationMixin) **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Idefics2CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics2ForConditionalGeneration`). - Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The hidden states of the image encoder after modality projection and perceiver resampling. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics2ForConditionalGeneration`). + Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only + computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 4e1a00fa98..90e15821a3 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -29,22 +29,13 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, logging from ..auto import AutoModel from .configuration_idefics3 import Idefics3Config, Idefics3VisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Idefics3Config" - @dataclass class Idefics3BaseModelOutputWithPast(ModelOutput): @@ -504,27 +495,7 @@ class Idefics3Connector(nn.Module): return image_hidden_states -IDEFICS3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Idefics3Config`] or [`Idefics3VisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Idefics3 Model outputting raw hidden-states without any specific head on top.", - IDEFICS3_START_DOCSTRING, -) +@auto_docstring class Idefics3PreTrainedModel(PreTrainedModel): config_class = Idefics3Config base_model_prefix = "model" @@ -555,26 +526,10 @@ class Idefics3PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -IDEFICS3_VISION_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Idefics3VisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The Idefics3 Vision Transformer Model outputting raw image embedding.", - IDEFICS3_VISION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Idefics3 Vision Transformer Model outputting raw image embedding. + """ ) class Idefics3VisionTransformer(Idefics3PreTrainedModel): config_class = Idefics3VisionConfig @@ -658,83 +613,10 @@ class Idefics3VisionTransformer(Idefics3PreTrainedModel): ) -IDEFICS3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`CLIPImageProcessor`] for processing images). - pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): - Mask to avoid performing attention on padding pixel indices. - image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The hidden states of the image encoder after modality projection. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """Idefics3 model consisting of a SIGLIP vision encoder and Llama3 language decoder""", - IDEFICS3_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Idefics3 model consisting of a SIGLIP vision encoder and Llama3 language decoder + """ ) class Idefics3Model(Idefics3PreTrainedModel): def __init__(self, config: Idefics3Config): @@ -821,8 +703,8 @@ class Idefics3Model(Idefics3PreTrainedModel): return new_inputs_embeds @can_return_tuple - @add_start_docstrings_to_model_forward( - """ + @auto_docstring( + custom_intro=""" Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where max_num_images is the maximum number of images among the batch_size samples in the batch. @@ -830,8 +712,7 @@ class Idefics3Model(Idefics3PreTrainedModel): For efficiency, we only pass through the vision_model's forward the real images by discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3. - """, - IDEFICS3_INPUTS_DOCSTRING, + """ ) def forward( self, @@ -850,6 +731,12 @@ class Idefics3Model(Idefics3PreTrainedModel): return_dict: Optional[bool] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, Idefics3BaseModelOutputWithPast]: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The hidden states of the image encoder after modality projection. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -961,9 +848,10 @@ class Idefics3Model(Idefics3PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The Idefics3 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. """, - IDEFICS3_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Idefics3 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. + """ ) class Idefics3ForConditionalGeneration(Idefics3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -1017,8 +905,7 @@ class Idefics3ForConditionalGeneration(Idefics3PreTrainedModel, GenerationMixin) self.lm_head = new_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(IDEFICS3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Idefics3CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1039,19 +926,15 @@ class Idefics3ForConditionalGeneration(Idefics3PreTrainedModel, GenerationMixin) **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Idefics3CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics3ForConditionalGeneration`). - Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - Returns: + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The hidden states of the image encoder after modality projection. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics3ForConditionalGeneration`). + Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only + computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/ijepa/modeling_ijepa.py b/src/transformers/models/ijepa/modeling_ijepa.py index c48310e425..956987290a 100644 --- a/src/transformers/models/ijepa/modeling_ijepa.py +++ b/src/transformers/models/ijepa/modeling_ijepa.py @@ -15,25 +15,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_ijepa import IJepaConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/ijepa_vith14_1k" - -# General docstring -_CONFIG_FOR_DOC = "IJepaConfig" - - class IJepaPatchEmbeddings(nn.Module): """ This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial @@ -154,12 +142,8 @@ class IJepaEmbeddings(nn.Module): return embeddings +@auto_docstring class IJepaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = IJepaConfig base_model_prefix = "ijepa" main_input_name = "pixel_values" @@ -479,51 +463,15 @@ class IJepaPooler(nn.Module): return pooled_output -IJEPA_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`IJepaImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -_EXPECTED_OUTPUT_SHAPE = [1, 256, 1280] - -IJEPA_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`IJepaConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare IJepa Model transformer outputting raw hidden-states without any specific head on top.", - IJEPA_START_DOCSTRING, -) +@auto_docstring class IJepaModel(IJepaPreTrainedModel): def __init__(self, config: IJepaConfig, add_pooling_layer: bool = False, use_mask_token: bool = False): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config self.embeddings = IJepaEmbeddings(config, use_mask_token=use_mask_token) @@ -546,14 +494,7 @@ class IJepaModel(IJepaPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(IJEPA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -616,12 +557,8 @@ class IJepaModel(IJepaPreTrainedModel): ) -_IMAGE_CLASS_CHECKPOINT = "facebook/ijepa_vith14_1k" -_IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states) e.g. for ImageNet. @@ -632,8 +569,7 @@ _IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" position embeddings to the higher resolution. - """, - IJEPA_START_DOCSTRING, + """ ) class IJepaForImageClassification(IJepaPreTrainedModel): def __init__(self, config: IJepaConfig) -> None: @@ -648,13 +584,7 @@ class IJepaForImageClassification(IJepaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(IJEPA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/ijepa/modular_ijepa.py b/src/transformers/models/ijepa/modular_ijepa.py index 2cf0fe32bf..cf31e6ec85 100644 --- a/src/transformers/models/ijepa/modular_ijepa.py +++ b/src/transformers/models/ijepa/modular_ijepa.py @@ -8,18 +8,8 @@ from transformers.models.ijepa.configuration_ijepa import IJepaConfig from ...modeling_outputs import ImageClassifierOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - torch_int, -) -from ..vit.modeling_vit import ( - ViTEmbeddings, - ViTForImageClassification, - ViTModel, -) - - -_CHECKPOINT_FOR_DOC = "facebook/ijepa_vith14_1k" +from ...utils import auto_docstring, torch_int +from ..vit.modeling_vit import ViTEmbeddings, ViTForImageClassification, ViTModel class IJepaEmbeddings(ViTEmbeddings): @@ -96,12 +86,8 @@ class IJepaEmbeddings(ViTEmbeddings): return embeddings +@auto_docstring class IJepaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = IJepaConfig base_model_prefix = "ijepa" main_input_name = "pixel_values" @@ -133,37 +119,21 @@ class IJepaPreTrainedModel(PreTrainedModel): module.mask_token.data.zero_() -_EXPECTED_OUTPUT_SHAPE = [1, 256, 1280] - -IJEPA_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`IJepaConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare IJepa Model transformer outputting raw hidden-states without any specific head on top.", - IJEPA_START_DOCSTRING, -) class IJepaModel(IJepaPreTrainedModel, ViTModel): def __init__(self, config: IJepaConfig, add_pooling_layer: bool = False, use_mask_token: bool = False): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config self.embeddings = IJepaEmbeddings(config, use_mask_token=use_mask_token) -_IMAGE_CLASS_CHECKPOINT = "facebook/ijepa_vith14_1k" -_IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states) e.g. for ImageNet. @@ -174,8 +144,7 @@ _IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" position embeddings to the higher resolution. - """, - IJEPA_START_DOCSTRING, + """ ) class IJepaForImageClassification(IJepaPreTrainedModel, ViTForImageClassification): def __init__(self, config: IJepaConfig): diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index ae56f7812a..68f6a04c5a 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -35,10 +35,8 @@ from ...modeling_outputs import ( from ...modeling_utils import PreTrainedModel from ...pytorch_utils import Conv1D, find_pruneable_heads_and_indices, prune_conv1d_layer from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, torch_float, ) from .configuration_imagegpt import ImageGPTConfig @@ -46,9 +44,6 @@ from .configuration_imagegpt import ImageGPTConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "openai/imagegpt-small" -_CONFIG_FOR_DOC = "ImageGPTConfig" - def load_tf_weights_in_imagegpt(model, config, imagegpt_checkpoint_path): """ @@ -482,12 +477,8 @@ class ImageGPTBlock(nn.Module): return outputs # hidden_states, present, (attentions, cross_attentions) +@auto_docstring class ImageGPTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ImageGPTConfig load_tf_weights = load_tf_weights_in_imagegpt base_model_prefix = "transformer" @@ -525,89 +516,7 @@ class ImageGPTPreTrainedModel(PreTrainedModel): p.data.normal_(mean=0.0, std=(self.config.initializer_range / math.sqrt(2 * self.config.n_layer))) -IMAGEGPT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ImageGPTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -IMAGEGPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else - `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input - sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. - - past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`): - Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see - `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have - their past given to this model should not be passed as `input_ids` as they have already been computed. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, input_ids_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ImageGPT Model transformer outputting raw hidden-states without any specific head on top.", - IMAGEGPT_START_DOCSTRING, -) +@auto_docstring class ImageGPTModel(ImageGPTPreTrainedModel): def __init__(self, config: ImageGPTConfig): super().__init__(config) @@ -641,8 +550,7 @@ class ImageGPTModel(ImageGPTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].attn.prune_heads(heads) - @add_start_docstrings_to_model_forward(IMAGEGPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -661,12 +569,15 @@ class ImageGPTModel(ImageGPTPreTrainedModel): **kwargs: Any, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: r""" - labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. - Returns: + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. Examples: @@ -871,12 +782,11 @@ class ImageGPTModel(ImageGPTPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The ImageGPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - IMAGEGPT_START_DOCSTRING, + """ ) class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -898,8 +808,7 @@ class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(IMAGEGPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -919,13 +828,20 @@ class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel, GenerationMixin): **kwargs: Any, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -1033,12 +949,11 @@ class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The ImageGPT Model transformer with an image classification head on top (linear layer). [`ImageGPTForImageClassification`] average-pools the hidden states in order to do the classification. - """, - IMAGEGPT_START_DOCSTRING, + """ ) class ImageGPTForImageClassification(ImageGPTPreTrainedModel): def __init__(self, config: ImageGPTConfig): @@ -1050,8 +965,7 @@ class ImageGPTForImageClassification(ImageGPTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(IMAGEGPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1069,13 +983,20 @@ class ImageGPTForImageClassification(ImageGPTPreTrainedModel): **kwargs: Any, ) -> Union[Tuple, SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoImageProcessor`]. See [`ImageGPTImageProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/informer/modeling_informer.py b/src/transformers/models/informer/modeling_informer.py index a67950233f..98acd1bd13 100644 --- a/src/transformers/models/informer/modeling_informer.py +++ b/src/transformers/models/informer/modeling_informer.py @@ -31,14 +31,15 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...time_series_utils import NegativeBinomialOutput, NormalOutput, StudentTOutput -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import ( + auto_docstring, + logging, +) from .configuration_informer import InformerConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "InformerConfig" - # Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesFeatureEmbedder with TimeSeries->Informer class InformerFeatureEmbedder(nn.Module): @@ -872,6 +873,7 @@ class InformerDecoderLayer(nn.Module): return outputs +@auto_docstring class InformerPreTrainedModel(PreTrainedModel): config_class = InformerConfig base_model_prefix = "model" @@ -892,177 +894,6 @@ class InformerPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -INFORMER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TimeSeriesTransformerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -INFORMER_INPUTS_DOCSTRING = r""" - Args: - past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`): - Past values of the time series, that serve as context in order to predict the future. The sequence size of - this tensor must be larger than the `context_length` of the model, since the model will use the larger size - to construct lag features, i.e. additional values from the past which are added in order to serve as "extra - context". - - The `sequence_length` here is equal to `config.context_length` + `max(config.lags_sequence)`, which if no - `lags_sequence` is configured, is equal to `config.context_length` + 7 (as by default, the largest - look-back index in `config.lags_sequence` is 7). The property `_past_length` returns the actual length of - the past. - - The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as - `static_categorical_features`, `static_real_features`, `past_time_features` and lags). - - Optionally, missing values need to be replaced with zeros and indicated via the `past_observed_mask`. - - For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of - variates in the time series per time step. - past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`): - Required time features, which the model internally will add to `past_values`. These could be things like - "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These - could also be so-called "age" features, which basically help the model know "at which point in life" a - time-series is. Age features have small values for distant past time steps and increase monotonically the - more we approach the current time step. Holiday features are also a good example of time features. - - These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where - the position encodings are learned from scratch internally as parameters of the model, the Time Series - Transformer requires to provide additional time features. The Time Series Transformer only learns - additional embeddings for `static_categorical_features`. - - Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features - must but known at prediction time. - - The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. - past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): - Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in - `[0, 1]`: - - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - - static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): - Optional static categorical features for which the model will learn an embedding, which it will add to the - values of the time series. - - Static categorical features are features which have the same value for all time steps (static over time). - - A typical example of a static categorical feature is a time series ID. - static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): - Optional static real features which the model will add to the values of the time series. - - Static real features are features which have the same value for all time steps (static over time). - - A typical example of a static real feature is promotion information. - future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)` or `(batch_size, prediction_length, input_size)`, *optional*): - Future values of the time series, that serve as labels for the model. The `future_values` is what the - Transformer needs during training to learn to output, given the `past_values`. - - The sequence length here is equal to `prediction_length`. - - See the demo notebook and code snippets for details. - - Optionally, during training any missing values need to be replaced with zeros and indicated via the - `future_observed_mask`. - - For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of - variates in the time series per time step. - future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`): - Required time features for the prediction window, which the model internally will add to `future_values`. - These could be things like "month of year", "day of the month", etc. encoded as vectors (for instance as - Fourier features). These could also be so-called "age" features, which basically help the model know "at - which point in life" a time-series is. Age features have small values for distant past time steps and - increase monotonically the more we approach the current time step. Holiday features are also a good example - of time features. - - These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where - the position encodings are learned from scratch internally as parameters of the model, the Time Series - Transformer requires to provide additional time features. The Time Series Transformer only learns - additional embeddings for `static_categorical_features`. - - Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features - must but known at prediction time. - - The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. - future_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): - Boolean mask to indicate which `future_values` were observed and which were missing. Mask values selected - in `[0, 1]`: - - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - - This mask is used to filter out missing values for the final loss calculation. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on certain token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Mask to avoid performing attention on certain token indices. By default, a causal mask will be used, to - make sure the model can only look at previous inputs in order to predict the future. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class InformerEncoder(InformerPreTrainedModel): """ Informer encoder consisting of *config.encoder_layers* self attention layers with distillation layers. Each @@ -1432,10 +1263,7 @@ class InformerDecoder(InformerPreTrainedModel): ) -@add_start_docstrings( - "The bare Informer Model outputting raw hidden-states without any specific head on top.", - INFORMER_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesTransformerModel with TimeSeriesTransformer->Informer,TIME_SERIES_TRANSFORMER->INFORMER,time-series-transformer->informer,TimeSeries->Informer class InformerModel(InformerPreTrainedModel): def __init__(self, config: InformerConfig): @@ -1575,8 +1403,7 @@ class InformerModel(InformerPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(INFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqTSModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1598,7 +1425,98 @@ class InformerModel(InformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Seq2SeqTSModelOutput, Tuple]: r""" - Returns: + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`): + Past values of the time series, that serve as context in order to predict the future. The sequence size of + this tensor must be larger than the `context_length` of the model, since the model will use the larger size + to construct lag features, i.e. additional values from the past which are added in order to serve as "extra + context". + + The `sequence_length` here is equal to `config.context_length` + `max(config.lags_sequence)`, which if no + `lags_sequence` is configured, is equal to `config.context_length` + 7 (as by default, the largest + look-back index in `config.lags_sequence` is 7). The property `_past_length` returns the actual length of + the past. + + The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as + `static_categorical_features`, `static_real_features`, `past_time_features` and lags). + + Optionally, missing values need to be replaced with zeros and indicated via the `past_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`): + Required time features, which the model internally will add to `past_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. Holiday features are also a good example of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in + `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): + Optional static categorical features for which the model will learn an embedding, which it will add to the + values of the time series. + + Static categorical features are features which have the same value for all time steps (static over time). + + A typical example of a static categorical feature is a time series ID. + static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): + Optional static real features which the model will add to the values of the time series. + + Static real features are features which have the same value for all time steps (static over time). + + A typical example of a static real feature is promotion information. + future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)` or `(batch_size, prediction_length, input_size)`, *optional*): + Future values of the time series, that serve as labels for the model. The `future_values` is what the + Transformer needs during training to learn to output, given the `past_values`. + + The sequence length here is equal to `prediction_length`. + + See the demo notebook and code snippets for details. + + Optionally, during training any missing values need to be replaced with zeros and indicated via the + `future_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`): + Required time features for the prediction window, which the model internally will add to `future_values`. + These could be things like "month of year", "day of the month", etc. encoded as vectors (for instance as + Fourier features). These could also be so-called "age" features, which basically help the model know "at + which point in life" a time-series is. Age features have small values for distant past time steps and + increase monotonically the more we approach the current time step. Holiday features are also a good example + of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. Examples: @@ -1694,10 +1612,7 @@ class InformerModel(InformerPreTrainedModel): ) -@add_start_docstrings( - "The Informer Model with a distribution head on top for time-series forecasting.", - INFORMER_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.time_series_transformer.modeling_time_series_transformer.TimeSeriesTransformerForPrediction with TimeSeriesTransformer->Informer,TIME_SERIES_TRANSFORMER->INFORMER,time-series-transformer->informer class InformerForPrediction(InformerPreTrainedModel): def __init__(self, config: InformerConfig): @@ -1739,8 +1654,7 @@ class InformerForPrediction(InformerPreTrainedModel): sliced_params = [p[:, -trailing_n:] for p in params] return self.distribution_output.distribution(sliced_params, loc=loc, scale=scale) - @add_start_docstrings_to_model_forward(INFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqTSModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1763,7 +1677,106 @@ class InformerForPrediction(InformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Seq2SeqTSModelOutput, Tuple]: r""" - Returns: + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`): + Past values of the time series, that serve as context in order to predict the future. The sequence size of + this tensor must be larger than the `context_length` of the model, since the model will use the larger size + to construct lag features, i.e. additional values from the past which are added in order to serve as "extra + context". + + The `sequence_length` here is equal to `config.context_length` + `max(config.lags_sequence)`, which if no + `lags_sequence` is configured, is equal to `config.context_length` + 7 (as by default, the largest + look-back index in `config.lags_sequence` is 7). The property `_past_length` returns the actual length of + the past. + + The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as + `static_categorical_features`, `static_real_features`, `past_time_features` and lags). + + Optionally, missing values need to be replaced with zeros and indicated via the `past_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`): + Required time features, which the model internally will add to `past_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. Holiday features are also a good example of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in + `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): + Optional static categorical features for which the model will learn an embedding, which it will add to the + values of the time series. + + Static categorical features are features which have the same value for all time steps (static over time). + + A typical example of a static categorical feature is a time series ID. + static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): + Optional static real features which the model will add to the values of the time series. + + Static real features are features which have the same value for all time steps (static over time). + + A typical example of a static real feature is promotion information. + future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)` or `(batch_size, prediction_length, input_size)`, *optional*): + Future values of the time series, that serve as labels for the model. The `future_values` is what the + Transformer needs during training to learn to output, given the `past_values`. + + The sequence length here is equal to `prediction_length`. + + See the demo notebook and code snippets for details. + + Optionally, during training any missing values need to be replaced with zeros and indicated via the + `future_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`): + Required time features for the prediction window, which the model internally will add to `future_values`. + These could be things like "month of year", "day of the month", etc. encoded as vectors (for instance as + Fourier features). These could also be so-called "age" features, which basically help the model know "at + which point in life" a time-series is. Age features have small values for distant past time steps and + increase monotonically the more we approach the current time step. Holiday features are also a good example + of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + future_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `future_values` were observed and which were missing. Mask values selected + in `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + + This mask is used to filter out missing values for the final loss calculation. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. Examples: diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index ec051387d6..d5aa8efc46 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -34,24 +34,13 @@ from ...modeling_outputs import ( from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - LossKwargs, - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import LossKwargs, ModelOutput, auto_docstring, can_return_tuple, logging, torch_int from ..auto import AutoModel, AutoModelForCausalLM, AutoModelForSeq2SeqLM from .configuration_instructblip import InstructBlipConfig, InstructBlipQFormerConfig, InstructBlipVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Salesforce/instructblip-flan-t5-xl" - @dataclass # Copied from transformers.models.blip_2.modeling_blip_2.Blip2ForConditionalGenerationModelOutput with Blip2->InstructBlip @@ -336,12 +325,8 @@ class InstructBlipEncoderLayer(nn.Module): return outputs +@auto_docstring class InstructBlipPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = InstructBlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True @@ -380,107 +365,6 @@ class InstructBlipPreTrainedModel(PreTrainedModel): module.query_tokens.data.zero_() -INSTRUCTBLIP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`InstructBlipConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -INSTRUCTBLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`InstructBlipProcessor`]. See - [`InstructBlipProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - -INSTRUCTBLIP_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`InstructBlipProcessor`]. See - [`InstructBlipProcessor.__call__`] for details. - - qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided - to serve as text prompt, which the Q-Former model will encode. - - Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for - details. - - [What are input IDs?](../glossary#input-ids) - - qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be - provided to serve as text prompt, which the language model can continue. - - Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for - details. - - [What are input IDs?](../glossary#input-ids) - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an - encoder-decoder language model (like T5) is used. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) - - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - Only relevant in case an encoder-decoder language model (like T5) is used. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). -""" - - # Copied from transformers.models.blip.modeling_blip.BlipEncoder with Blip->InstructBlip class InstructBlipEncoder(nn.Module): """ @@ -584,8 +468,7 @@ class InstructBlipVisionModel(InstructBlipPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(INSTRUCTBLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=InstructBlipVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -594,10 +477,6 @@ class InstructBlipVisionModel(InstructBlipPreTrainedModel): return_dict: Optional[bool] = None, interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1319,11 +1198,10 @@ class InstructBlipQFormerModel(InstructBlipPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" InstructBLIP base Model consisting of language model, qformer and vision encoder. - """, - INSTRUCTBLIP_START_DOCSTRING, + """ ) class InstructBlipModel(InstructBlipPreTrainedModel): main_input_name = "pixel_values" @@ -1380,7 +1258,7 @@ class InstructBlipModel(InstructBlipPreTrainedModel): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility @can_return_tuple - @add_start_docstrings_to_model_forward(INSTRUCTBLIP_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1397,6 +1275,36 @@ class InstructBlipModel(InstructBlipPreTrainedModel): use_cache: Optional[bool] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, InstructBlipForConditionalGenerationModelOutput]: + r""" + qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided + to serve as text prompt, which the Q-Former model will encode. + + Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict # step 1: forward the images through the vision encoder, @@ -1470,15 +1378,14 @@ class InstructBlipModel(InstructBlipPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision encoder, Querying Transformer (Q-Former) and a language model. One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token. - """, - INSTRUCTBLIP_START_DOCSTRING, + """ ) class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, GenerationMixin): config_class = InstructBlipConfig @@ -1560,10 +1467,7 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati self.language_model._hf_hook.io_same_device = True # For `generate` compatibility @can_return_tuple - @add_start_docstrings_to_model_forward(INSTRUCTBLIP_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=InstructBlipForConditionalGenerationModelOutput, config_class=InstructBlipVisionConfig - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1582,13 +1486,39 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, Generati **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, InstructBlipForConditionalGenerationModelOutput]: r""" + qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided + to serve as text prompt, which the Q-Former model will encode. + + Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 4e78afaa88..48eaa4fa85 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -38,16 +38,7 @@ from ...modeling_outputs import ( from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - LossKwargs, - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import LossKwargs, ModelOutput, auto_docstring, can_return_tuple, logging, torch_int from ..auto import AutoModel, AutoModelForCausalLM, AutoModelForSeq2SeqLM from .configuration_instructblipvideo import ( InstructBlipVideoConfig, @@ -390,24 +381,6 @@ class InstructBlipVideoEncoder(nn.Module): ) -INSTRUCTBLIPVIDEO_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`InstructBlipVideoProcessor`]. See - [`InstructBlipVideoProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - - class InstructBlipVideoQFormerMultiHeadAttention(nn.Module): def __init__(self, config, is_cross_attention=False): super().__init__() @@ -885,96 +858,8 @@ class InstructBlipVideoQFormerEmbeddings(nn.Module): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -INSTRUCTBLIPVIDEO_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`InstructBlipVideoConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -INSTRUCTBLIPVIDEO_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`InstructBlipVideoProcessor`]. See - [`InstructBlipVideoProcessor.__call__`] for details. - - qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided - to serve as text prompt, which the Q-Former model will encode. - - Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for - details. - - [What are input IDs?](../glossary#input-ids) - - qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be - provided to serve as text prompt, which the language model can continue. - - Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for - details. - - [What are input IDs?](../glossary#input-ids) - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an - encoder-decoder language model (like T5) is used. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) - - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - Only relevant in case an encoder-decoder language model (like T5) is used. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). -""" - - +@auto_docstring class InstructBlipVideoPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = InstructBlipVideoConfig base_model_prefix = "blip" supports_gradient_checkpointing = True @@ -1028,8 +913,7 @@ class InstructBlipVideoVisionModel(InstructBlipVideoPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(INSTRUCTBLIPVIDEO_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=InstructBlipVideoVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1038,10 +922,6 @@ class InstructBlipVideoVisionModel(InstructBlipVideoPreTrainedModel): return_dict: Optional[bool] = None, interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1314,11 +1194,10 @@ class InstructBlipVideoForConditionalGenerationModelOutput(ModelOutput): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" InstructBlipVideo base Model consisting of language model, qformer and vision encoder. - """, - INSTRUCTBLIPVIDEO_START_DOCSTRING, + """ ) class InstructBlipVideoModel(InstructBlipVideoPreTrainedModel): main_input_name = "pixel_values" @@ -1375,7 +1254,7 @@ class InstructBlipVideoModel(InstructBlipVideoPreTrainedModel): self.language_model._hf_hook.io_same_device = True # For `generate` compatibility @can_return_tuple - @add_start_docstrings_to_model_forward(INSTRUCTBLIPVIDEO_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1392,6 +1271,36 @@ class InstructBlipVideoModel(InstructBlipVideoPreTrainedModel): use_cache: Optional[bool] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, InstructBlipVideoForConditionalGenerationModelOutput]: + r""" + qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided + to serve as text prompt, which the Q-Former model will encode. + + Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict # step 1: forward the images through the vision encoder, @@ -1475,15 +1384,14 @@ class InstructBlipVideoModel(InstructBlipVideoPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision encoder, Querying Transformer (Q-Former) and a language model. One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token. - """, - INSTRUCTBLIPVIDEO_START_DOCSTRING, + """ ) class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel, GenerationMixin): config_class = InstructBlipVideoConfig @@ -1563,10 +1471,7 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel self.language_model._hf_hook.io_same_device = True # For `generate` compatibility @can_return_tuple - @add_start_docstrings_to_model_forward(INSTRUCTBLIPVIDEO_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=InstructBlipVideoForConditionalGenerationModelOutput, config_class=InstructBlipVideoConfig - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1585,13 +1490,39 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, InstructBlipVideoForConditionalGenerationModelOutput]: r""" + qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided + to serve as text prompt, which the Q-Former model will encode. + + Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python diff --git a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py index 566b00d983..44a0022ee4 100644 --- a/src/transformers/models/instructblipvideo/modular_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modular_instructblipvideo.py @@ -37,7 +37,7 @@ from ...configuration_utils import PretrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES from ...processing_utils import Unpack -from ...utils import add_start_docstrings_to_model_forward, can_return_tuple, logging, replace_return_docstrings +from ...utils import logging from ..auto import CONFIG_MAPPING, AutoConfig @@ -194,12 +194,7 @@ class InstructBlipVideoForConditionalGenerationModelOutput(InstructBlipForCondit pass -INSTRUCTBLIPVIDEO_INPUTS_DOCSTRING = None - - class InstructBlipVideoModel(InstructBlipModel): - @can_return_tuple - @add_start_docstrings_to_model_forward(INSTRUCTBLIPVIDEO_INPUTS_DOCSTRING) def forward( self, pixel_values: torch.FloatTensor, @@ -300,11 +295,6 @@ class InstructBlipVideoModel(InstructBlipModel): class InstructBlipVideoForConditionalGeneration(InstructBlipForConditionalGeneration): - @can_return_tuple - @add_start_docstrings_to_model_forward(INSTRUCTBLIPVIDEO_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=InstructBlipVideoForConditionalGenerationModelOutput, config_class=InstructBlipVideoConfig - ) def forward( self, pixel_values: torch.FloatTensor, diff --git a/src/transformers/models/internvl/modeling_internvl.py b/src/transformers/models/internvl/modeling_internvl.py index 2b1683e497..a6005133fc 100644 --- a/src/transformers/models/internvl/modeling_internvl.py +++ b/src/transformers/models/internvl/modeling_internvl.py @@ -37,13 +37,10 @@ from ...processing_utils import Unpack from ...utils import ( LossKwargs, ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging, - replace_return_docstrings, torch_int, ) from ..auto import AutoModel @@ -53,12 +50,6 @@ from .configuration_internvl import InternVLConfig, InternVLVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "OpenGVLab/InternVL3-1B-hf" - - -_CONFIG_FOR_DOC = "InternVLConfig" - - @use_kernel_forward_from_hub("RMSNorm") class InternVLVisionRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -188,12 +179,8 @@ class InternVLVisionAttention(nn.Module): return outputs +@auto_docstring class InternVLVisionPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = InternVLVisionConfig base_model_prefix = "internvl_vision" main_input_name = "pixel_values" @@ -495,42 +482,7 @@ class InternVLVisionEncoder(nn.Module): ) -_EXPECTED_OUTPUT_SHAPE = [1, 197, 768] - -_CONFIG_VISION_FOR_DOC = "InternVLVisionConfig" - - -INTERNVL_VISION_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`InternVLVisionConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -INTERNVL_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`InternVLVisionImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare InternVLVision Model transformer outputting raw hidden-states without any specific head on top.", - INTERNVL_VISION_START_DOCSTRING, -) +@auto_docstring class InternVLVisionModel(InternVLVisionPreTrainedModel): def __init__(self, config: InternVLVisionConfig) -> None: super().__init__(config) @@ -550,14 +502,7 @@ class InternVLVisionModel(InternVLVisionPreTrainedModel): return self.embeddings.patch_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(INTERNVL_VISION_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=InternVLVisionModelOutputWithPooling, - config_class=_CONFIG_VISION_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -591,27 +536,7 @@ class InternVLVisionModel(InternVLVisionPreTrainedModel): ) -INTERNVL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`InternVLConfig`] or [`InternVLVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare InternVL Model outputting raw hidden-states without any specific head on top.", - INTERNVL_START_DOCSTRING, -) +@auto_docstring class InternVLPreTrainedModel(PreTrainedModel): config_class = InternVLConfig base_model_prefix = "" @@ -687,86 +612,10 @@ class InternVLModelOutputWithPast(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None -INTERNVL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`InternVLProcessor`] uses - [`CLIPImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The InternVL model which consists of a vision backbone and a language model, without a language modeling head.""", - INTERNVL_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The InternVL model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class InternVLModel(InternVLPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -831,7 +680,7 @@ class InternVLModel(InternVLPreTrainedModel): return vision_features @can_return_tuple - @add_start_docstrings_to_model_forward(INTERNVL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -997,9 +846,10 @@ class InternVLCausalLMOutputWithPast(ModelOutput): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The INTERNVL model which consists of a vision backbone and a language model.""", - INTERNVL_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The INTERNVL model which consists of a vision backbone and a language model. + """ ) class InternVLForConditionalGeneration(InternVLPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -1042,8 +892,7 @@ class InternVLForConditionalGeneration(InternVLPreTrainedModel, GenerationMixin) return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(INTERNVL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=InternVLCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1065,23 +914,13 @@ class InternVLForConditionalGeneration(InternVLPreTrainedModel, GenerationMixin) **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, InternVLCausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: + ```python >>> import torch >>> from transformers import AutoProcessor, AutoModelForImageTextToText diff --git a/src/transformers/models/internvl/modular_internvl.py b/src/transformers/models/internvl/modular_internvl.py index bc3c3bfc6d..71fdcdf9d4 100644 --- a/src/transformers/models/internvl/modular_internvl.py +++ b/src/transformers/models/internvl/modular_internvl.py @@ -27,15 +27,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, can_return_tuple, logging, torch_int from ..clip.modeling_clip import CLIPMLP from ..janus.modeling_janus import JanusVisionAttention from ..llama.modeling_llama import LlamaRMSNorm @@ -51,11 +43,6 @@ from .configuration_internvl import InternVLConfig, InternVLVisionConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "OpenGVLab/InternVL3-1B-hf" - -_CONFIG_VISION_FOR_DOC = "InternVLVisionConfig" - - def eager_attention_forward( module: nn.Module, query: torch.Tensor, @@ -149,12 +136,8 @@ class InternVLVisionAttention(JanusVisionAttention): return outputs +@auto_docstring class InternVLVisionPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = InternVLVisionConfig base_model_prefix = "internvl_vision" main_input_name = "pixel_values" @@ -445,42 +428,7 @@ class InternVLVisionEncoder(nn.Module): ) -_EXPECTED_OUTPUT_SHAPE = [1, 197, 768] - -_CONFIG_VISION_FOR_DOC = "InternVLVisionConfig" - - -INTERNVL_VISION_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`InternVLVisionConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -INTERNVL_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`InternVLVisionImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare InternVLVision Model transformer outputting raw hidden-states without any specific head on top.", - INTERNVL_VISION_START_DOCSTRING, -) +@auto_docstring class InternVLVisionModel(InternVLVisionPreTrainedModel): def __init__(self, config: InternVLVisionConfig) -> None: super().__init__(config) @@ -500,14 +448,7 @@ class InternVLVisionModel(InternVLVisionPreTrainedModel): return self.embeddings.patch_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(INTERNVL_VISION_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=InternVLVisionModelOutputWithPooling, - config_class=_CONFIG_VISION_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -541,9 +482,6 @@ class InternVLVisionModel(InternVLVisionPreTrainedModel): ) -_CONFIG_FOR_DOC = "InternVLConfig" - - class InternVLPreTrainedModel(LlavaPreTrainedModel): def _init_weights(self, module): std = getattr(self.config, "initializer_range", self.config.get_text_config().initializer_range) @@ -665,28 +603,10 @@ class InternVLCausalLMOutputWithPast(LlavaCausalLMOutputWithPast): class InternVLForConditionalGeneration(LlavaForConditionalGeneration): - @can_return_tuple - @add_start_docstrings_to_model_forward(INTERNVL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=InternVLCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward(**super_kwargs): r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: - Example: + ```python >>> import torch >>> from transformers import AutoProcessor, AutoModelForImageTextToText diff --git a/src/transformers/models/jamba/modeling_jamba.py b/src/transformers/models/jamba/modeling_jamba.py index b72b6f0b53..90127ba70f 100755 --- a/src/transformers/models/jamba/modeling_jamba.py +++ b/src/transformers/models/jamba/modeling_jamba.py @@ -30,27 +30,12 @@ from torch import nn from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache # we need __iter__ and __len__ of pkv from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - AttentionMaskConverter, -) +from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available -from ...modeling_outputs import ( - MoeCausalLMOutputWithPast, - MoeModelOutputWithPast, - SequenceClassifierOutputWithPast, -) +from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast, SequenceClassifierOutputWithPast from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) -from ...utils.import_utils import ( - is_causal_conv1d_available, - is_mamba_ssm_available, -) +from ...utils import auto_docstring, can_return_tuple, logging +from ...utils.import_utils import is_causal_conv1d_available, is_mamba_ssm_available from .configuration_jamba import JambaConfig @@ -76,8 +61,6 @@ is_fast_path_available = all( logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "JambaConfig" - # Copied from transformers.models.mixtral.modeling_mixtral.load_balancing_loss_func with gate->router def load_balancing_loss_func( @@ -1072,27 +1055,7 @@ class JambaMambaDecoderLayer(nn.Module): return outputs -JAMBA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`JambaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Jamba Model outputting raw hidden-states without any specific head on top.", - JAMBA_START_DOCSTRING, -) +@auto_docstring class JambaPreTrainedModel(PreTrainedModel): config_class = JambaConfig base_model_prefix = "model" @@ -1123,85 +1086,11 @@ class JambaPreTrainedModel(PreTrainedModel): module.D.data.fill_(1.0) -JAMBA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`HybridMambaAttentionDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - A HybridMambaAttentionDynamicCache object containing pre-computed hidden-states (keys and values in the - self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`. - Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and - `(batch_size, d_inner, d_state)` respectively. - See the `HybridMambaAttentionDynamicCache` class for more details. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - ALL_DECODER_LAYER_TYPES = {"attention": JambaAttentionDecoderLayer, "mamba": JambaMambaDecoderLayer} -@add_start_docstrings( - "The bare Jamba Model outputting raw hidden-states without any specific head on top.", - JAMBA_START_DOCSTRING, -) # Adapted from transformers.models.mistral.modeling_mistral.MistralModel with MISTRAL->JAMBA, Mistral->Jamba +@auto_docstring class JambaModel(JambaPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`JambaDecoderLayer`] @@ -1236,7 +1125,7 @@ class JambaModel(JambaPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(JAMBA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1433,8 +1322,7 @@ class JambaForCausalLM(JambaPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(JAMBA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1452,19 +1340,10 @@ class JambaForCausalLM(JambaPreTrainedModel, GenerationMixin): **loss_kwargs, ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1596,8 +1475,8 @@ class JambaForCausalLM(JambaPreTrainedModel, GenerationMixin): return model_inputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Jamba Model with a sequence classification head on top (linear layer). [`JambaForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1608,8 +1487,7 @@ class JambaForCausalLM(JambaPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - JAMBA_START_DOCSTRING, + """ ) # Copied from transformers.models.mixtral.modeling_mixtral.MixtralForSequenceClassification with Mixtral->Jamba, MIXTRAL->JAMBA, BaseModelOutputWithPast->MoeModelOutputWithPast class JambaForSequenceClassification(JambaPreTrainedModel): @@ -1629,7 +1507,7 @@ class JambaForSequenceClassification(JambaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(JAMBA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/janus/modeling_janus.py b/src/transformers/models/janus/modeling_janus.py index 853e371c89..a5ba0337bb 100644 --- a/src/transformers/models/janus/modeling_janus.py +++ b/src/transformers/models/janus/modeling_janus.py @@ -36,12 +36,10 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, Mod from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, is_torch_available, logging, - replace_return_docstrings, torch_int, ) from ..auto import AutoModel @@ -54,31 +52,8 @@ if is_torch_available(): logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "JanusConfig" - -JANUS_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`JanusConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Janus Model outputting raw hidden-states without any specific head on top.", - JANUS_START_DOCSTRING, -) +@auto_docstring class JanusPreTrainedModel(PreTrainedModel): config_class = JanusConfig base_model_prefix = "model" @@ -560,24 +535,7 @@ class JanusVisionEncoder(nn.Module): ) -JANUS_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`JanusProcessor`]. See [`JanusProcessor.__call__`] for - details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. -""" - - +@auto_docstring class JanusVisionModel(JanusPreTrainedModel): main_input_name = "pixel_values" config_class = JanusVisionConfig @@ -593,8 +551,7 @@ class JanusVisionModel(JanusPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(JANUS_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=JanusVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -603,10 +560,6 @@ class JanusVisionModel(JanusPreTrainedModel): return_dict: Optional[bool] = None, interpolate_pos_encoding: bool = False, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -995,35 +948,15 @@ class JanusVQVAEDecoder(nn.Module): return hidden_state -JANUS_VQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`JanusVQVAEConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens. +@auto_docstring( + custom_intro=""" + The VQ-VAE model used in Janus for encoding/decoding images into discrete tokens. This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv Taigman](https://arxiv.org/abs/2203.13131). - """, - JANUS_VQ_START_DOCSTRING, + """ ) class JanusVQVAE(JanusPreTrainedModel): - """Vision Transformer-based VQ-VAE model for encoding and decoding pixel values.""" - config_class = JanusVQVAEConfig - _no_split_modules = [ "JanusVQVAEAttnBlock", "JanusVQVAEResnetBlock", @@ -1071,21 +1004,11 @@ class JanusVQVAE(JanusPreTrainedModel): return pixel_values @can_return_tuple + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, ) -> Tuple[torch.FloatTensor, torch.FloatTensor]: - """ - Encodes pixel values into quantized tokens and decodes them back. - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. - Returns: - decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - Reconstructed pixel values after encoding and decoding the input. - embedding_loss (`torch.FloatTensor`): Embedding loss. - """ - batch_size = pixel_values.shape[0] quant, embedding_loss, indices = self.encode(pixel_values) decoded_pixel_values = self.decode(indices.view(batch_size, -1)) @@ -1128,76 +1051,10 @@ class JanusVQVAEHead(nn.Module): return hidden_states -JANUS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.""", - JANUS_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model. + """ ) class JanusModel(JanusPreTrainedModel): def __init__(self, config: JanusConfig): @@ -1233,7 +1090,7 @@ class JanusModel(JanusPreTrainedModel): return image_embeds @can_return_tuple - @add_start_docstrings_to_model_forward(JANUS_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1346,8 +1203,7 @@ class JanusForConditionalGeneration(JanusPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(JANUS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=JanusCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1365,20 +1221,10 @@ class JanusForConditionalGeneration(JanusPreTrainedModel, GenerationMixin): **kwargs, ): r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( diff --git a/src/transformers/models/janus/modular_janus.py b/src/transformers/models/janus/modular_janus.py index c9dec37ad1..6ff52d7558 100644 --- a/src/transformers/models/janus/modular_janus.py +++ b/src/transformers/models/janus/modular_janus.py @@ -25,18 +25,10 @@ from transformers.models.blip.image_processing_blip import BlipImageProcessor from ...activations import ACT2FN from ...cache_utils import Cache -from ...generation import ( - ClassifierFreeGuidanceLogitsProcessor, - GenerationMixin, - GenerationMode, - LogitsProcessorList, -) +from ...generation import ClassifierFreeGuidanceLogitsProcessor, GenerationMixin, GenerationMode, LogitsProcessorList from ...generation.utils import GenerateDecoderOnlyOutput from ...image_processing_utils import BatchFeature, get_size_dict -from ...image_transforms import ( - resize, - to_channel_dimension_format, -) +from ...image_transforms import resize, to_channel_dimension_format from ...image_utils import ( ChannelDimension, ImageInput, @@ -50,15 +42,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_available, - is_vision_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_available, is_vision_available, logging from ..auto import AutoModel from ..blip_2.modeling_blip_2 import Blip2VisionModel from ..chameleon.configuration_chameleon import ChameleonVQVAEConfig @@ -73,11 +57,7 @@ from ..chameleon.modeling_chameleon import ( from ..idefics.modeling_idefics import IdeficsBaseModelOutputWithPast, IdeficsCausalLMOutputWithPast from ..llama.modeling_llama import eager_attention_forward from ..siglip.configuration_siglip import SiglipVisionConfig -from ..siglip.modeling_siglip import ( - SiglipEncoder, - SiglipEncoderLayer, - SiglipVisionEmbeddings, -) +from ..siglip.modeling_siglip import SiglipEncoder, SiglipEncoderLayer, SiglipVisionEmbeddings if is_torch_available(): @@ -89,7 +69,6 @@ if is_torch_available(): if is_vision_available(): import PIL - from ...configuration_utils import PretrainedConfig from ..auto import CONFIG_MAPPING, AutoConfig @@ -97,7 +76,6 @@ from ..auto import CONFIG_MAPPING, AutoConfig logger = logging.get_logger(__name__) # General docstring -_CONFIG_FOR_DOC = "JanusConfig" class JanusVisionConfig(SiglipVisionConfig): @@ -396,27 +374,7 @@ class JanusConfig(PretrainedConfig): super().__init__(**kwargs) -JANUS_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`JanusConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Janus Model outputting raw hidden-states without any specific head on top.", - JANUS_START_DOCSTRING, -) +@auto_docstring class JanusPreTrainedModel(PreTrainedModel): config_class = JanusConfig base_model_prefix = "model" @@ -849,8 +807,6 @@ class JanusVQVAEDecoder(nn.Module): class JanusVQVAE(ChameleonVQVAE): - """Vision Transformer-based VQ-VAE model for encoding and decoding pixel values.""" - _no_split_modules = [ "JanusVQVAEAttnBlock", "JanusVQVAEResnetBlock", @@ -886,21 +842,11 @@ class JanusVQVAE(ChameleonVQVAE): return pixel_values @can_return_tuple + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, ) -> Tuple[torch.FloatTensor, torch.FloatTensor]: - """ - Encodes pixel values into quantized tokens and decodes them back. - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. - Returns: - decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - Reconstructed pixel values after encoding and decoding the input. - embedding_loss (`torch.FloatTensor`): Embedding loss. - """ - batch_size = pixel_values.shape[0] quant, embedding_loss, indices = self.encode(pixel_values) decoded_pixel_values = self.decode(indices.view(batch_size, -1)) @@ -943,76 +889,10 @@ class JanusVQVAEHead(nn.Module): return hidden_states -JANUS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.""", - JANUS_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model. + """ ) class JanusModel(JanusPreTrainedModel): def __init__(self, config: JanusConfig): @@ -1048,7 +928,7 @@ class JanusModel(JanusPreTrainedModel): return image_embeds @can_return_tuple - @add_start_docstrings_to_model_forward(JANUS_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1161,8 +1041,7 @@ class JanusForConditionalGeneration(JanusPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(JANUS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=JanusCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1180,20 +1059,10 @@ class JanusForConditionalGeneration(JanusPreTrainedModel, GenerationMixin): **kwargs, ): r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( diff --git a/src/transformers/models/jetmoe/modeling_jetmoe.py b/src/transformers/models/jetmoe/modeling_jetmoe.py index ca8d55f630..e241898bd3 100644 --- a/src/transformers/models/jetmoe/modeling_jetmoe.py +++ b/src/transformers/models/jetmoe/modeling_jetmoe.py @@ -27,21 +27,10 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available -from ...modeling_outputs import ( - MoeCausalLMOutputWithPast, - MoeModelOutputWithPast, - SequenceClassifierOutputWithPast, -) +from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast, SequenceClassifierOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_jetmoe import JetMoeConfig @@ -56,9 +45,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "jetmoe" -_CONFIG_FOR_DOC = "JetMoeConfig" - # Copied from transformers.models.mixtral.modeling_mixtral.load_balancing_loss_func def load_balancing_loss_func( @@ -828,12 +814,8 @@ class JetMoeBlock(nn.Module): return outputs +@auto_docstring class JetMoePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = JetMoeConfig base_model_prefix = "transformer" supports_gradient_checkpointing = False @@ -865,64 +847,7 @@ class JetMoePreTrainedModel(PreTrainedModel): module.bias.data.zero_() -JETMOE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`JetMoeConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -JETMOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare JetMoe Model outputting raw hidden-states without any specific head on top.", - JETMOE_START_DOCSTRING, -) +@auto_docstring class JetMoeModel(JetMoePreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`JetMoeBlock`] @@ -955,7 +880,7 @@ class JetMoeModel(JetMoePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(JETMOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1254,8 +1179,7 @@ class JetMoeForCausalLM(JetMoePreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(JETMOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1273,19 +1197,10 @@ class JetMoeForCausalLM(JetMoePreTrainedModel, GenerationMixin): **kwargs, ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -1352,8 +1267,8 @@ class JetMoeForCausalLM(JetMoePreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The JetMoe Model transformer with a sequence classification head on top (linear layer). [`JetMoeForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1364,8 +1279,7 @@ class JetMoeForCausalLM(JetMoePreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - JETMOE_START_DOCSTRING, + """ ) # Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->JetMoe, LLAMA->JETMOE, BaseModelOutputWithPast->MoeModelOutputWithPast class JetMoeForSequenceClassification(JetMoePreTrainedModel): @@ -1385,7 +1299,7 @@ class JetMoeForSequenceClassification(JetMoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(JETMOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/kosmos2/modeling_kosmos2.py b/src/transformers/models/kosmos2/modeling_kosmos2.py index b5eed09549..980996b282 100644 --- a/src/transformers/models/kosmos2/modeling_kosmos2.py +++ b/src/transformers/models/kosmos2/modeling_kosmos2.py @@ -33,23 +33,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import LossKwargs, ModelOutput, auto_docstring, can_return_tuple, logging, torch_int from .configuration_kosmos2 import Kosmos2Config, Kosmos2TextConfig, Kosmos2VisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = Kosmos2Config - def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): """ @@ -99,179 +88,6 @@ def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_l return incremental_indices.long() + padding_idx -KOSMOS2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Kosmos2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -KOSMOS2_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -KOSMOS2_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - image_embeds: (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*): - Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`. - image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0, - 1]`: - - - 1 for places where to put the image features, - - 0 for places that are not for image features (i.e. for text tokens). - - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -KOSMOS2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`CLIPImageProcessor.__call__`] for details. - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0, - 1]`: - - - 1 for places where to put the image features, - - 0 for places that are not for image features (i.e. for text tokens). - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - image_embeds: (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*): - Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - @dataclass class Kosmos2ModelOutput(ModelOutput): """ @@ -1377,12 +1193,8 @@ class Kosmos2TextTransformer(nn.Module): ) +@auto_docstring class Kosmos2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Kosmos2Config supports_gradient_checkpointing = True _no_split_modules = ["Kosmos2VisionEncoderLayer", "Kosmos2TextBlock"] @@ -1489,8 +1301,7 @@ class Kosmos2VisionModel(Kosmos2PreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(KOSMOS2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Kosmos2VisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1499,10 +1310,6 @@ class Kosmos2VisionModel(Kosmos2PreTrainedModel): interpolate_pos_encoding: bool = False, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ return self.model( pixel_values=pixel_values, output_attentions=output_attentions, @@ -1528,8 +1335,7 @@ class Kosmos2TextModel(Kosmos2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(KOSMOS2_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=Kosmos2TextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1550,8 +1356,19 @@ class Kosmos2TextModel(Kosmos2PreTrainedModel): **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: r""" - Returns: + image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0, + 1]`: + - 1 for places where to put the image features, + - 0 for places that are not for image features (i.e. for text tokens). + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*): + Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`. """ return self.model( input_ids=input_ids, @@ -1576,12 +1393,11 @@ class Kosmos2TextModel(Kosmos2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The text model from KOSMOS-2 with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - KOSMOS2_START_DOCSTRING, + """ ) class Kosmos2TextForCausalLM(Kosmos2PreTrainedModel, GenerationMixin): config_class = Kosmos2TextConfig @@ -1609,8 +1425,7 @@ class Kosmos2TextForCausalLM(Kosmos2PreTrainedModel, GenerationMixin): self.lm_head = new_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(KOSMOS2_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=Kosmos2TextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1632,13 +1447,23 @@ class Kosmos2TextForCausalLM(Kosmos2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" + image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0, + 1]`: + + - 1 for places where to put the image features, + - 0 for places that are not for image features (i.e. for text tokens). + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - - Returns: - + image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*): + Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1775,11 +1600,10 @@ class Kosmos2ImageToTextProjection(nn.Module): return hidden_states, attn_weights -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" KOSMOS-2 Model for generating text and image features. The model consists of a vision encoder and a language model. - """, - KOSMOS2_START_DOCSTRING, + """ ) class Kosmos2Model(Kosmos2PreTrainedModel): config_class = Kosmos2Config @@ -1802,8 +1626,7 @@ class Kosmos2Model(Kosmos2PreTrainedModel): self.text_model.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(KOSMOS2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Kosmos2ModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1823,7 +1646,14 @@ class Kosmos2Model(Kosmos2PreTrainedModel): **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, Kosmos2ModelOutput]: r""" - Returns: + image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0, + 1]`: + + - 1 for places where to put the image features, + - 0 for places that are not for image features (i.e. for text tokens). + image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*): + Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`. Examples: @@ -1907,12 +1737,11 @@ class Kosmos2Model(Kosmos2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" KOSMOS-2 Model for generating text and bounding boxes given an image. The model consists of a vision encoder and a language model. - """, - KOSMOS2_START_DOCSTRING, + """ ) class Kosmos2ForConditionalGeneration(Kosmos2PreTrainedModel, GenerationMixin): config_class = Kosmos2Config @@ -1943,8 +1772,7 @@ class Kosmos2ForConditionalGeneration(Kosmos2PreTrainedModel, GenerationMixin): self.text_model.set_output_embeddings(new_embeddings) @can_return_tuple - @add_start_docstrings_to_model_forward(KOSMOS2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Kosmos2ForConditionalGenerationModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1964,12 +1792,18 @@ class Kosmos2ForConditionalGeneration(Kosmos2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Kosmos2ForConditionalGenerationModelOutput]: r""" + image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0, + 1]`: + + - 1 for places where to put the image features, + - 0 for places that are not for image features (i.e. for text tokens). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - - Returns: + image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*): + Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`. Examples: diff --git a/src/transformers/models/layoutlm/modeling_layoutlm.py b/src/transformers/models/layoutlm/modeling_layoutlm.py index 8c31521a3f..68d05e509f 100644 --- a/src/transformers/models/layoutlm/modeling_layoutlm.py +++ b/src/transformers/models/layoutlm/modeling_layoutlm.py @@ -33,15 +33,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_layoutlm import LayoutLMConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LayoutLMConfig" -_CHECKPOINT_FOR_DOC = "microsoft/layoutlm-base-uncased" - LayoutLMLayerNorm = nn.LayerNorm @@ -610,12 +607,8 @@ class LayoutLMOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class LayoutLMPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LayoutLMConfig base_model_prefix = "layoutlm" supports_gradient_checkpointing = True @@ -639,72 +632,7 @@ class LayoutLMPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -LAYOUTLM_START_DOCSTRING = r""" - The LayoutLM model was proposed in [LayoutLM: Pre-training of Text and Layout for Document Image - Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei and - Ming Zhou. - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`LayoutLMConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LAYOUTLM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for - tokens that are NOT MASKED, `0` for MASKED tokens. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1` - indicates the head is **not masked**, `0` indicates the head is **masked**. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - If set to `True`, the model will return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare LayoutLM Model transformer outputting raw hidden-states without any specific head on top.", - LAYOUTLM_START_DOCSTRING, -) +@auto_docstring class LayoutLMModel(LayoutLMPreTrainedModel): def __init__(self, config): super(LayoutLMModel, self).__init__(config) @@ -731,8 +659,7 @@ class LayoutLMModel(LayoutLMPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(LAYOUTLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -749,7 +676,11 @@ class LayoutLMModel(LayoutLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: r""" - Returns: + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. Examples: @@ -853,7 +784,7 @@ class LayoutLMModel(LayoutLMPreTrainedModel): ) -@add_start_docstrings("""LayoutLM Model with a `language modeling` head on top.""", LAYOUTLM_START_DOCSTRING) +@auto_docstring class LayoutLMForMaskedLM(LayoutLMPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -876,8 +807,7 @@ class LayoutLMForMaskedLM(LayoutLMPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(LAYOUTLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -895,13 +825,16 @@ class LayoutLMForMaskedLM(LayoutLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MaskedLMOutput]: r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -979,12 +912,11 @@ class LayoutLMForMaskedLM(LayoutLMPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LayoutLM Model with a sequence classification head on top (a linear layer on top of the pooled output) e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. - """, - LAYOUTLM_START_DOCSTRING, + """ ) class LayoutLMForSequenceClassification(LayoutLMPreTrainedModel): def __init__(self, config): @@ -1000,8 +932,7 @@ class LayoutLMForSequenceClassification(LayoutLMPreTrainedModel): def get_input_embeddings(self): return self.layoutlm.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(LAYOUTLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1017,13 +948,16 @@ class LayoutLMForSequenceClassification(LayoutLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1115,13 +1049,12 @@ class LayoutLMForSequenceClassification(LayoutLMPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LayoutLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for sequence labeling (information extraction) tasks such as the [FUNSD](https://guillaumejaume.github.io/FUNSD/) dataset and the [SROIE](https://rrc.cvc.uab.es/?ch=13) dataset. - """, - LAYOUTLM_START_DOCSTRING, + """ ) class LayoutLMForTokenClassification(LayoutLMPreTrainedModel): def __init__(self, config): @@ -1137,8 +1070,7 @@ class LayoutLMForTokenClassification(LayoutLMPreTrainedModel): def get_input_embeddings(self): return self.layoutlm.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(LAYOUTLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1154,11 +1086,14 @@ class LayoutLMForTokenClassification(LayoutLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Examples: ```python @@ -1233,16 +1168,13 @@ class LayoutLMForTokenClassification(LayoutLMPreTrainedModel): ) -@add_start_docstrings( - """ - LayoutLM Model with a span classification head on top for extractive question-answering tasks such as - [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the final hidden-states output to compute `span - start logits` and `span end logits`). - """, - LAYOUTLM_START_DOCSTRING, -) +@auto_docstring class LayoutLMForQuestionAnswering(LayoutLMPreTrainedModel): def __init__(self, config, has_visual_segment_embedding=True): + r""" + has_visual_segment_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to add visual segment embeddings. + """ super().__init__(config) self.num_labels = config.num_labels @@ -1255,7 +1187,7 @@ class LayoutLMForQuestionAnswering(LayoutLMPreTrainedModel): def get_input_embeddings(self): return self.layoutlm.embeddings.word_embeddings - @replace_return_docstrings(output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1272,16 +1204,11 @@ class LayoutLMForQuestionAnswering(LayoutLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - - Returns: + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. Example: diff --git a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2_fast.py b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2_fast.py index 937ceed3a2..418685892f 100644 --- a/src/transformers/models/layoutlmv2/image_processing_layoutlmv2_fast.py +++ b/src/transformers/models/layoutlmv2/image_processing_layoutlmv2_fast.py @@ -16,19 +16,13 @@ from typing import Optional, Union -from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - BaseImageProcessorFast, - BatchFeature, - DefaultFastImageProcessorKwargs, -) +from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs from ...image_transforms import ChannelDimension, group_images_by_shape, reorder_images from ...image_utils import ImageInput, PILImageResampling, SizeDict from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -51,15 +45,8 @@ if is_torchvision_available(): class LayoutLMv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): - apply_ocr: Optional[bool] - ocr_lang: Optional[str] - tesseract_config: Optional[str] - - -@add_start_docstrings( - "Constructs a fast LayoutLMv2 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, """ + Args: apply_ocr (`bool`, *optional*, defaults to `True`): Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by the `apply_ocr` parameter in the `preprocess` method. @@ -70,8 +57,14 @@ class LayoutLMv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): Any additional custom configuration flags that are forwarded to the `config` parameter when calling Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the `preprocess` method. - """, -) + """ + + apply_ocr: Optional[bool] + ocr_lang: Optional[str] + tesseract_config: Optional[str] + + +@auto_docstring class LayoutLMv2ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR size = {"height": 224, "width": 224} @@ -85,21 +78,7 @@ class LayoutLMv2ImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[LayoutLMv2FastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - apply_ocr (`bool`, *optional*, defaults to `True`): - Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by - the `apply_ocr` parameter in the `preprocess` method. - ocr_lang (`str`, *optional*): - The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is - used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method. - tesseract_config (`str`, *optional*): - Any additional custom configuration flags that are forwarded to the `config` parameter when calling - Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the - `preprocess` method. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[LayoutLMv2FastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py index 8cb9cbdf95..4333e2c4c8 100755 --- a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py @@ -32,14 +32,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_detectron2_available, - logging, - replace_return_docstrings, - requires_backends, -) +from ...utils import auto_docstring, is_detectron2_available, logging, requires_backends from .configuration_layoutlmv2 import LayoutLMv2Config @@ -54,9 +47,6 @@ if is_detectron2_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/layoutlmv2-base-uncased" -_CONFIG_FOR_DOC = "LayoutLMv2Config" - class LayoutLMv2Embeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings.""" @@ -490,12 +480,8 @@ class LayoutLMv2Encoder(nn.Module): ) +@auto_docstring class LayoutLMv2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LayoutLMv2Config base_model_prefix = "layoutlmv2" @@ -613,77 +599,6 @@ class LayoutLMv2VisualBackbone(nn.Module): self.backbone = my_convert_sync_batchnorm(self.backbone, process_group=sync_bn_groups[node_rank]) -LAYOUTLMV2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`LayoutLMv2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LAYOUTLMV2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `{0}`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. - - image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`): - Batch of document images. - - attention_mask (`torch.FloatTensor` of shape `{0}`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `{0}`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `{0}`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class LayoutLMv2Pooler(nn.Module): def __init__(self, config): super().__init__() @@ -699,10 +614,7 @@ class LayoutLMv2Pooler(nn.Module): return pooled_output -@add_start_docstrings( - "The bare LayoutLMv2 Model transformer outputting raw hidden-states without any specific head on top.", - LAYOUTLMV2_START_DOCSTRING, -) +@auto_docstring class LayoutLMv2Model(LayoutLMv2PreTrainedModel): def __init__(self, config): requires_backends(self, "detectron2") @@ -813,8 +725,7 @@ class LayoutLMv2Model(LayoutLMv2PreTrainedModel): else: raise ValueError("You have to specify either input_ids or inputs_embeds") - @add_start_docstrings_to_model_forward(LAYOUTLMV2_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -830,7 +741,13 @@ class LayoutLMv2Model(LayoutLMv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Return: + bbox (`torch.LongTensor` of shape `((batch_size, sequence_length), 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`): + Batch of document images. Examples: @@ -955,14 +872,13 @@ class LayoutLMv2Model(LayoutLMv2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LayoutLMv2 Model with a sequence classification head on top (a linear layer on top of the concatenation of the final hidden state of the [CLS] token, average-pooled initial visual embeddings and average-pooled final visual embeddings, e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. - """, - LAYOUTLMV2_START_DOCSTRING, + """ ) class LayoutLMv2ForSequenceClassification(LayoutLMv2PreTrainedModel): def __init__(self, config): @@ -978,8 +894,7 @@ class LayoutLMv2ForSequenceClassification(LayoutLMv2PreTrainedModel): def get_input_embeddings(self): return self.layoutlmv2.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(LAYOUTLMV2_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -996,13 +911,38 @@ class LayoutLMv2ForSequenceClassification(LayoutLMv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `batch_size, sequence_length`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`): + Batch of document images. + token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Example: ```python @@ -1137,14 +1077,13 @@ class LayoutLMv2ForSequenceClassification(LayoutLMv2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LayoutLMv2 Model with a token classification head on top (a linear layer on top of the text part of the hidden states) e.g. for sequence labeling (information extraction) tasks such as [FUNSD](https://guillaumejaume.github.io/FUNSD/), [SROIE](https://rrc.cvc.uab.es/?ch=13), [CORD](https://github.com/clovaai/cord) and [Kleister-NDA](https://github.com/applicaai/kleister-nda). - """, - LAYOUTLMV2_START_DOCSTRING, + """ ) class LayoutLMv2ForTokenClassification(LayoutLMv2PreTrainedModel): def __init__(self, config): @@ -1160,8 +1099,7 @@ class LayoutLMv2ForTokenClassification(LayoutLMv2PreTrainedModel): def get_input_embeddings(self): return self.layoutlmv2.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(LAYOUTLMV2_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1178,11 +1116,36 @@ class LayoutLMv2ForTokenClassification(LayoutLMv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `batch_size, sequence_length`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`): + Batch of document images. + token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Example: ```python @@ -1269,16 +1232,13 @@ class LayoutLMv2ForTokenClassification(LayoutLMv2PreTrainedModel): ) -@add_start_docstrings( - """ - LayoutLMv2 Model with a span classification head on top for extractive question-answering tasks such as - [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the text part of the hidden-states output to - compute `span start logits` and `span end logits`). - """, - LAYOUTLMV2_START_DOCSTRING, -) +@auto_docstring class LayoutLMv2ForQuestionAnswering(LayoutLMv2PreTrainedModel): def __init__(self, config, has_visual_segment_embedding=True): + r""" + has_visual_segment_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to add visual segment embeddings. + """ super().__init__(config) self.num_labels = config.num_labels config.has_visual_segment_embedding = has_visual_segment_embedding @@ -1291,8 +1251,7 @@ class LayoutLMv2ForQuestionAnswering(LayoutLMv2PreTrainedModel): def get_input_embeddings(self): return self.layoutlmv2.embeddings.word_embeddings - @add_start_docstrings_to_model_forward(LAYOUTLMV2_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1310,16 +1269,33 @@ class LayoutLMv2ForQuestionAnswering(LayoutLMv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `batch_size, sequence_length`): + Indices of input sequence tokens in the vocabulary. - Returns: + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`): + Batch of document images. + token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) Example: diff --git a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3_fast.py b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3_fast.py index 0507a4f729..35ce04b183 100644 --- a/src/transformers/models/layoutlmv3/image_processing_layoutlmv3_fast.py +++ b/src/transformers/models/layoutlmv3/image_processing_layoutlmv3_fast.py @@ -17,8 +17,6 @@ from typing import Optional, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -28,7 +26,7 @@ from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, ImageI from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -51,15 +49,8 @@ if is_torchvision_available(): class LayoutLMv3FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): - apply_ocr: Optional[bool] - ocr_lang: Optional[str] - tesseract_config: Optional[str] - - -@add_start_docstrings( - "Constructs a fast LayoutLMv3 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, """ + Args: apply_ocr (`bool`, *optional*, defaults to `True`): Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by the `apply_ocr` parameter in the `preprocess` method. @@ -70,8 +61,14 @@ class LayoutLMv3FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): Any additional custom configuration flags that are forwarded to the `config` parameter when calling Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the `preprocess` method. - """, -) + """ + + apply_ocr: Optional[bool] + ocr_lang: Optional[str] + tesseract_config: Optional[str] + + +@auto_docstring class LayoutLMv3ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN @@ -88,21 +85,7 @@ class LayoutLMv3ImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[LayoutLMv3FastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - apply_ocr (`bool`, *optional*, defaults to `True`): - Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by - the `apply_ocr` parameter in the `preprocess` method. - ocr_lang (`str`, *optional*): - The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is - used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method. - tesseract_config (`str`, *optional*): - Any additional custom configuration flags that are forwarded to the `config` parameter when calling - Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the - `preprocess` method. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[LayoutLMv3FastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py b/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py index d55a9d9b0e..3605001d8a 100644 --- a/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py +++ b/src/transformers/models/layoutlmv3/modeling_layoutlmv3.py @@ -34,10 +34,8 @@ from ...modeling_outputs import ( from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, torch_int, ) from .configuration_layoutlmv3 import LayoutLMv3Config @@ -45,157 +43,6 @@ from .configuration_layoutlmv3 import LayoutLMv3Config logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LayoutLMv3Config" - - -LAYOUTLMV3_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`LayoutLMv3Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LAYOUTLMV3_MODEL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Batch of document images. Each image is divided into patches of shape `(num_channels, config.patch_size, - config.patch_size)` and the total number of patches (=`patch_sequence_length`) equals to `((height / - config.patch_size) * (width / config.patch_size))`. - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -LAYOUTLMV3_DOWNSTREAM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Batch of document images. Each image is divided into patches of shape `(num_channels, config.patch_size, - config.patch_size)` and the total number of patches (=`patch_sequence_length`) equals to `((height / - config.patch_size) * (width / config.patch_size))`. - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - class LayoutLMv3PatchEmbeddings(nn.Module): """LayoutLMv3 image (patch) embeddings. This class also automatically interpolates the position embeddings for varying @@ -350,12 +197,8 @@ class LayoutLMv3TextEmbeddings(nn.Module): return embeddings +@auto_docstring class LayoutLMv3PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LayoutLMv3Config base_model_prefix = "layoutlmv3" @@ -746,10 +589,7 @@ class LayoutLMv3Output(nn.Module): return hidden_states -@add_start_docstrings( - "The bare LayoutLMv3 Model transformer outputting raw hidden-states without any specific head on top.", - LAYOUTLMV3_START_DOCSTRING, -) +@auto_docstring class LayoutLMv3Model(LayoutLMv3PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -839,10 +679,7 @@ class LayoutLMv3Model(LayoutLMv3PreTrainedModel): return embeddings - @add_start_docstrings_to_model_forward( - LAYOUTLMV3_MODEL_INPUTS_DOCSTRING.format("batch_size, token_sequence_length") - ) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -858,7 +695,47 @@ class LayoutLMv3Model(LayoutLMv3PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, token_sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + bbox (`torch.LongTensor` of shape `(batch_size, token_sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + token_type_ids (`torch.LongTensor` of shape `(batch_size, token_sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, token_sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, token_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. Examples: @@ -1028,14 +905,13 @@ class LayoutLMv3ClassificationHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LayoutLMv3 Model with a token classification head on top (a linear layer on top of the final hidden states) e.g. for sequence labeling (information extraction) tasks such as [FUNSD](https://guillaumejaume.github.io/FUNSD/), [SROIE](https://rrc.cvc.uab.es/?ch=13), [CORD](https://github.com/clovaai/cord) and [Kleister-NDA](https://github.com/applicaai/kleister-nda). - """, - LAYOUTLMV3_START_DOCSTRING, + """ ) class LayoutLMv3ForTokenClassification(LayoutLMv3PreTrainedModel): def __init__(self, config): @@ -1051,10 +927,7 @@ class LayoutLMv3ForTokenClassification(LayoutLMv3PreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward( - LAYOUTLMV3_DOWNSTREAM_INPUTS_DOCSTRING.format("batch_size, sequence_length") - ) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1071,11 +944,14 @@ class LayoutLMv3ForTokenClassification(LayoutLMv3PreTrainedModel): pixel_values: Optional[torch.LongTensor] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Examples: ```python @@ -1141,14 +1017,7 @@ class LayoutLMv3ForTokenClassification(LayoutLMv3PreTrainedModel): ) -@add_start_docstrings( - """ - LayoutLMv3 Model with a span classification head on top for extractive question-answering tasks such as - [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the text part of the hidden-states output to - compute `span start logits` and `span end logits`). - """, - LAYOUTLMV3_START_DOCSTRING, -) +@auto_docstring class LayoutLMv3ForQuestionAnswering(LayoutLMv3PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1159,10 +1028,7 @@ class LayoutLMv3ForQuestionAnswering(LayoutLMv3PreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward( - LAYOUTLMV3_DOWNSTREAM_INPUTS_DOCSTRING.format("batch_size, sequence_length") - ) - @replace_return_docstrings(output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1180,16 +1046,11 @@ class LayoutLMv3ForQuestionAnswering(LayoutLMv3PreTrainedModel): pixel_values: Optional[torch.LongTensor] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - - Returns: + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. Examples: @@ -1271,13 +1132,12 @@ class LayoutLMv3ForQuestionAnswering(LayoutLMv3PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LayoutLMv3 Model with a sequence classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. - """, - LAYOUTLMV3_START_DOCSTRING, + """ ) class LayoutLMv3ForSequenceClassification(LayoutLMv3PreTrainedModel): def __init__(self, config): @@ -1289,10 +1149,7 @@ class LayoutLMv3ForSequenceClassification(LayoutLMv3PreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward( - LAYOUTLMV3_DOWNSTREAM_INPUTS_DOCSTRING.format("batch_size, sequence_length") - ) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1308,8 +1165,12 @@ class LayoutLMv3ForSequenceClassification(LayoutLMv3PreTrainedModel): bbox: Optional[torch.LongTensor] = None, pixel_values: Optional[torch.LongTensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: - """ - Returns: + r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. Examples: diff --git a/src/transformers/models/led/modeling_led.py b/src/transformers/models/led/modeling_led.py index be56ef1bb1..e6e21ce897 100755 --- a/src/transformers/models/led/modeling_led.py +++ b/src/transformers/models/led/modeling_led.py @@ -27,31 +27,14 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask -from ...modeling_outputs import ( - BaseModelOutputWithPastAndCrossAttentions, - Seq2SeqLMOutput, - Seq2SeqModelOutput, - Seq2SeqQuestionAnsweringModelOutput, - Seq2SeqSequenceClassifierOutput, -) +from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_led import LEDConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "allenai/led-base-16384" -_CONFIG_FOR_DOC = "LEDConfig" - def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): """ @@ -1119,6 +1102,7 @@ class LEDClassificationHead(nn.Module): return hidden_states +@auto_docstring class LEDPreTrainedModel(PreTrainedModel): config_class = LEDConfig base_model_prefix = "led" @@ -1462,161 +1446,6 @@ class LEDSeq2SeqQuestionAnsweringModelOutput(ModelOutput): encoder_global_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None -LED_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. See the superclass documentation for the generic methods the library - implements for all its models (such as downloading or saving, resizing the input embeddings, pruning heads etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for general usage and behavior. - - Parameters: - config ([`LEDConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LED_GENERATION_EXAMPLE = r""" - Summarization example: - - ```python - >>> import torch - >>> from transformers import AutoTokenizer, LEDForConditionalGeneration - - >>> model = LEDForConditionalGeneration.from_pretrained("allenai/led-large-16384-arxiv") - >>> tokenizer = AutoTokenizer.from_pretrained("allenai/led-large-16384-arxiv") - - >>> ARTICLE_TO_SUMMARIZE = '''Transformers (Vaswani et al., 2017) have achieved state-of-the-art - ... results in a wide range of natural language tasks including generative language modeling - ... (Dai et al., 2019; Radford et al., 2019) and discriminative ... language understanding (Devlin et al., 2019). - ... This success is partly due to the self-attention component which enables the network to capture contextual - ... information from the entire sequence. While powerful, the memory and computational requirements of - ... self-attention grow quadratically with sequence length, making it infeasible (or very expensive) to - ... process long sequences. To address this limitation, we present Longformer, a modified Transformer - ... architecture with a self-attention operation that scales linearly with the sequence length, making it - ... versatile for processing long documents (Fig 1). This is an advantage for natural language tasks such as - ... long document classification, question answering (QA), and coreference resolution, where existing approaches - ... partition or shorten the long context into smaller sequences that fall within the typical 512 token limit - ... of BERT-style pretrained models. Such partitioning could potentially result in loss of important - ... cross-partition information, and to mitigate this problem, existing methods often rely on complex - ... architectures to address such interactions. On the other hand, our proposed Longformer is able to build - ... contextual representations of the entire context using multiple layers of attention, reducing the need for - ... task-specific architectures.''' - >>> inputs = tokenizer.encode(ARTICLE_TO_SUMMARIZE, return_tensors="pt") - - >>> # Global attention on the first token (cf. Beltagy et al. 2020) - >>> global_attention_mask = torch.zeros_like(inputs) - >>> global_attention_mask[:, 0] = 1 - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs, global_attention_mask=global_attention_mask, num_beams=3, max_length=32) - >>> print(tokenizer.decode(summary_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)) - ``` -""" - -LED_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify - to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the - default strategy. - global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to decide the attention given on each token, local attention or global attention for the encoder. - Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is - important for task-specific finetuning because it makes the model more flexible at representing the task. - For example, for classification, the token should be given global attention. For QA, all question - tokens should also have global attention. Please refer to the [Longformer - paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`: - - - 0 for local attention (a sliding window attention), - - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class LEDEncoder(LEDPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self-attention layers. Each layer is a @@ -2171,10 +2000,7 @@ class LEDDecoder(LEDPreTrainedModel): ) -@add_start_docstrings( - "The bare LED Model outputting raw hidden-states without any specific head on top.", - LED_START_DOCSTRING, -) +@auto_docstring class LEDModel(LEDPreTrainedModel): _tied_weights_keys = ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"] @@ -2204,12 +2030,7 @@ class LEDModel(LEDPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2229,6 +2050,41 @@ class LEDModel(LEDPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], LEDSeq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify + to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the + default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention for the encoder. + Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is + important for task-specific finetuning because it makes the model more flexible at representing the task. + For example, for classification, the token should be given global attention. For QA, all question + tokens should also have global attention. Please refer to the [Longformer + paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -2296,8 +2152,10 @@ class LEDModel(LEDPreTrainedModel): ) -@add_start_docstrings( - "The LED Model with a language modeling head. Can be used for summarization.", LED_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The LED Model with a language modeling head. Can be used for summarization. + """ ) class LEDForConditionalGeneration(LEDPreTrainedModel, GenerationMixin): base_model_prefix = "led" @@ -2341,9 +2199,7 @@ class LEDForConditionalGeneration(LEDPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(LED_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2365,14 +2221,81 @@ class LEDForConditionalGeneration(LEDPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], LEDSeq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify + to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the + default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention for the encoder. + Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is + important for task-specific finetuning because it makes the model more flexible at representing the task. + For example, for classification, the token should be given global attention. For QA, all question + tokens should also have global attention. Please refer to the [Longformer + paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Summarization: - Conditional generation example: + ```python + >>> import torch + >>> from transformers import AutoTokenizer, LEDForConditionalGeneration + + >>> model = LEDForConditionalGeneration.from_pretrained("allenai/led-large-16384-arxiv") + >>> tokenizer = AutoTokenizer.from_pretrained("allenai/led-large-16384-arxiv") + + >>> ARTICLE_TO_SUMMARIZE = '''Transformers (Vaswani et al., 2017) have achieved state-of-the-art + ... results in a wide range of natural language tasks including generative language modeling + ... (Dai et al., 2019; Radford et al., 2019) and discriminative ... language understanding (Devlin et al., 2019). + ... This success is partly due to the self-attention component which enables the network to capture contextual + ... information from the entire sequence. While powerful, the memory and computational requirements of + ... self-attention grow quadratically with sequence length, making it infeasible (or very expensive) to + ... process long sequences. To address this limitation, we present Longformer, a modified Transformer + ... architecture with a self-attention operation that scales linearly with the sequence length, making it + ... versatile for processing long documents (Fig 1). This is an advantage for natural language tasks such as + ... long document classification, question answering (QA), and coreference resolution, where existing approaches + ... partition or shorten the long context into smaller sequences that fall within the typical 512 token limit + ... of BERT-style pretrained models. Such partitioning could potentially result in loss of important + ... cross-partition information, and to mitigate this problem, existing methods often rely on complex + ... architectures to address such interactions. On the other hand, our proposed Longformer is able to build + ... contextual representations of the entire context using multiple layers of attention, reducing the need for + ... task-specific architectures.''' + >>> inputs = tokenizer.encode(ARTICLE_TO_SUMMARIZE, return_tensors="pt") + + >>> # Global attention on the first token (cf. Beltagy et al. 2020) + >>> global_attention_mask = torch.zeros_like(inputs) + >>> global_attention_mask[:, 0] = 1 + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs, global_attention_mask=global_attention_mask, num_beams=3, max_length=32) + >>> print(tokenizer.decode(summary_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=True)) + ``` + + Example Conditional generation : ```python >>> from transformers import AutoTokenizer, LEDForConditionalGeneration @@ -2385,7 +2308,8 @@ class LEDForConditionalGeneration(LEDPreTrainedModel, GenerationMixin): >>> prediction = model.generate(input_ids)[0] >>> print(tokenizer.decode(prediction, skip_special_tokens=True)) - ```""" + ``` + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -2454,12 +2378,11 @@ class LEDForConditionalGeneration(LEDPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LED model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - LED_START_DOCSTRING, + """ ) class LEDForSequenceClassification(LEDPreTrainedModel): _tied_weights_keys = ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"] @@ -2483,12 +2406,7 @@ class LEDForSequenceClassification(LEDPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2509,6 +2427,39 @@ class LEDForSequenceClassification(LEDPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], LEDSeq2SeqSequenceClassifierOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify + to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the + default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention for the encoder. + Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is + important for task-specific finetuning because it makes the model more flexible at representing the task. + For example, for classification, the token should be given global attention. For QA, all question + tokens should also have global attention. Please refer to the [Longformer + paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). @@ -2590,13 +2541,7 @@ class LEDForSequenceClassification(LEDPreTrainedModel): ) -@add_start_docstrings( - """ - LED Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layer - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - LED_START_DOCSTRING, -) +@auto_docstring class LEDForQuestionAnswering(LEDPreTrainedModel): _tied_weights_keys = ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"] @@ -2612,12 +2557,7 @@ class LEDForQuestionAnswering(LEDPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqQuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2639,14 +2579,39 @@ class LEDForQuestionAnswering(LEDPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], LEDSeq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify + to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the + default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention for the encoder. + Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is + important for task-specific finetuning because it makes the model more flexible at representing the task. + For example, for classification, the token should be given global attention. For QA, all question + tokens should also have global attention. Please refer to the [Longformer + paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if start_positions is not None and end_positions is not None: diff --git a/src/transformers/models/levit/image_processing_levit_fast.py b/src/transformers/models/levit/image_processing_levit_fast.py index 87b0d0ba3e..9d8774ade8 100644 --- a/src/transformers/models/levit/image_processing_levit_fast.py +++ b/src/transformers/models/levit/image_processing_levit_fast.py @@ -14,13 +14,13 @@ # limitations under the License. """Fast Image processor class for LeViT.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, SizeDict +from ...image_processing_utils_fast import BaseImageProcessorFast, SizeDict from ...image_transforms import ( ChannelDimension, get_resize_output_image_size, ) from ...image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling -from ...utils import add_start_docstrings, is_torch_available, is_torchvision_available, is_torchvision_v2_available +from ...utils import auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available if is_torch_available(): @@ -33,10 +33,7 @@ if is_torchvision_available(): from torchvision.transforms import functional as F -@add_start_docstrings( - "Constructs a fast Levit image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class LevitImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = IMAGENET_DEFAULT_MEAN diff --git a/src/transformers/models/levit/modeling_levit.py b/src/transformers/models/levit/modeling_levit.py index ff09660d62..837d59979a 100644 --- a/src/transformers/models/levit/modeling_levit.py +++ b/src/transformers/models/levit/modeling_levit.py @@ -30,23 +30,12 @@ from ...modeling_outputs import ( ModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_levit import LevitConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "LevitConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/levit-128S" -_EXPECTED_OUTPUT_SHAPE = [1, 16, 384] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/levit-128S" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - @dataclass class LevitForImageClassificationWithTeacherOutput(ModelOutput): @@ -479,12 +468,8 @@ class LevitClassificationLayer(nn.Module): return logits +@auto_docstring class LevitPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LevitConfig base_model_prefix = "levit" main_input_name = "pixel_values" @@ -503,35 +488,7 @@ class LevitPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -LEVIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`LevitConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LEVIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`LevitImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Levit model outputting raw features without any specific head on top.", - LEVIT_START_DOCSTRING, -) +@auto_docstring class LevitModel(LevitPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -541,14 +498,7 @@ class LevitModel(LevitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LEVIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -585,12 +535,11 @@ class LevitModel(LevitPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Levit Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - LEVIT_START_DOCSTRING, + """ ) class LevitForImageClassification(LevitPreTrainedModel): def __init__(self, config): @@ -609,13 +558,7 @@ class LevitForImageClassification(LevitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LEVIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -670,14 +613,13 @@ class LevitForImageClassification(LevitPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LeViT Model transformer with image classification heads on top (a linear layer on top of the final hidden state and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet. .. warning:: This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet supported. - """, - LEVIT_START_DOCSTRING, + """ ) class LevitForImageClassificationWithTeacher(LevitPreTrainedModel): def __init__(self, config): @@ -701,13 +643,7 @@ class LevitForImageClassificationWithTeacher(LevitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LEVIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=LevitForImageClassificationWithTeacherOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, diff --git a/src/transformers/models/lilt/modeling_lilt.py b/src/transformers/models/lilt/modeling_lilt.py index b3d057fd64..9839e13f36 100644 --- a/src/transformers/models/lilt/modeling_lilt.py +++ b/src/transformers/models/lilt/modeling_lilt.py @@ -32,14 +32,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_lilt import LiltConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LiltConfig" - class LiltTextEmbeddings(nn.Module): def __init__(self, config): @@ -568,12 +566,8 @@ class LiltPooler(nn.Module): return pooled_output +@auto_docstring class LiltPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LiltConfig base_model_prefix = "lilt" supports_gradient_checkpointing = True @@ -596,84 +590,13 @@ class LiltPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -LILT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LiltConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LILT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare LiLT Model transformer outputting raw hidden-states without any specific head on top.", - LILT_START_DOCSTRING, -) +@auto_docstring class LiltModel(LiltPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -700,8 +623,7 @@ class LiltModel(LiltPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(LILT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -716,8 +638,11 @@ class LiltModel(LiltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPooling]: r""" - - Returns: + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. Examples: @@ -814,12 +739,11 @@ class LiltModel(LiltPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" LiLT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - LILT_START_DOCSTRING, + """ ) class LiltForSequenceClassification(LiltPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification.__init__ with Roberta->Lilt, roberta->lilt @@ -834,8 +758,7 @@ class LiltForSequenceClassification(LiltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LILT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -851,13 +774,16 @@ class LiltForSequenceClassification(LiltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -932,13 +858,7 @@ class LiltForSequenceClassification(LiltPreTrainedModel): ) -@add_start_docstrings( - """ - Lilt Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - LILT_START_DOCSTRING, -) +@auto_docstring class LiltForTokenClassification(LiltPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForTokenClassification.__init__ with Roberta->Lilt, roberta->lilt def __init__(self, config): @@ -955,8 +875,7 @@ class LiltForTokenClassification(LiltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LILT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -972,11 +891,14 @@ class LiltForTokenClassification(LiltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Examples: ```python @@ -1058,13 +980,7 @@ class LiltClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - Lilt Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - LILT_START_DOCSTRING, -) +@auto_docstring class LiltForQuestionAnswering(LiltPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForQuestionAnswering.__init__ with Roberta->Lilt, roberta->lilt def __init__(self, config): @@ -1077,8 +993,7 @@ class LiltForQuestionAnswering(LiltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LILT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1095,16 +1010,11 @@ class LiltForQuestionAnswering(LiltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - - Returns: + bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. Examples: diff --git a/src/transformers/models/llama/configuration_llama.py b/src/transformers/models/llama/configuration_llama.py index 066534f109..1ea1f12427 100644 --- a/src/transformers/models/llama/configuration_llama.py +++ b/src/transformers/models/llama/configuration_llama.py @@ -28,6 +28,7 @@ class LlamaConfig(PretrainedConfig): This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the LLaMA-7B. + e.g. [meta-llama/Llama-2-7b-hf](https://huggingface.co/meta-llama/Llama-2-7b-hf) Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index cad5502e3f..a25f05ea7c 100644 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -40,16 +40,7 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_llama import LlamaConfig @@ -63,9 +54,6 @@ from ...integrations import use_kernel_forward_from_hub logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "meta-llama/Llama-2-7b-hf" -_CONFIG_FOR_DOC = "LlamaConfig" - @use_kernel_forward_from_hub("RMSNorm") class LlamaRMSNorm(nn.Module): @@ -343,27 +331,7 @@ class LlamaDecoderLayer(GradientCheckpointingLayer): return outputs -LLAMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LlamaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAMA_START_DOCSTRING, -) +@auto_docstring class LlamaPreTrainedModel(PreTrainedModel): config_class = LlamaConfig base_model_prefix = "model" @@ -392,88 +360,8 @@ class LlamaPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -LLAMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAMA_START_DOCSTRING, -) +@auto_docstring class LlamaModel(LlamaPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] - - Args: - config: LlamaConfig - """ - def __init__(self, config: LlamaConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -497,7 +385,7 @@ class LlamaModel(LlamaPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -719,6 +607,7 @@ class LlamaModel(LlamaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class LlamaForCausalLM(LlamaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -752,8 +641,7 @@ class LlamaForCausalLM(LlamaPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -770,19 +658,10 @@ class LlamaForCausalLM(LlamaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -837,8 +716,8 @@ class LlamaForCausalLM(LlamaPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LLaMa Model transformer with a sequence classification head on top (linear layer). [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -849,8 +728,7 @@ class LlamaForCausalLM(LlamaPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - LLAMA_START_DOCSTRING, + """ ) class LlamaForSequenceClassification(LlamaPreTrainedModel): def __init__(self, config): @@ -869,7 +747,7 @@ class LlamaForSequenceClassification(LlamaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -938,13 +816,7 @@ class LlamaForSequenceClassification(LlamaPreTrainedModel): ) -@add_start_docstrings( - """ -The Llama Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - LLAMA_START_DOCSTRING, -) +@auto_docstring class LlamaForQuestionAnswering(LlamaPreTrainedModel): base_model_prefix = "transformer" @@ -964,7 +836,7 @@ class LlamaForQuestionAnswering(LlamaPreTrainedModel): self.transformer.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -978,17 +850,6 @@ class LlamaForQuestionAnswering(LlamaPreTrainedModel): output_hidden_states: Optional[bool] = None, **kwargs, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.transformer( input_ids, attention_mask=attention_mask, @@ -1019,13 +880,7 @@ class LlamaForQuestionAnswering(LlamaPreTrainedModel): ) -@add_start_docstrings( - """ - The Llama Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - LLAMA_START_DOCSTRING, -) +@auto_docstring class LlamaForTokenClassification(LlamaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1050,12 +905,7 @@ class LlamaForTokenClassification(LlamaPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/llama4/image_processing_llama4_fast.py b/src/transformers/models/llama4/image_processing_llama4_fast.py index fd9d68fb7b..63aa53bc60 100644 --- a/src/transformers/models/llama4/image_processing_llama4_fast.py +++ b/src/transformers/models/llama4/image_processing_llama4_fast.py @@ -21,8 +21,6 @@ from typing import List, Optional, Set, Tuple, Union from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -36,7 +34,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -109,11 +107,6 @@ def get_max_res_without_distortion( return new_height, new_width -class Llama4ImageProcessorKwargs(DefaultFastImageProcessorKwargs): - max_patches: Optional[int] - resize_to_max_canvas: Optional[bool] - - def split_to_tiles(images: torch.Tensor, num_tiles_height: int, num_tiles_width: int) -> torch.Tensor: # Split image into number of required tiles (width x height) batch_size, num_channels, height, width = images.size() @@ -330,20 +323,23 @@ def get_best_fit( return tuple(optimal_canvas.tolist()) -@add_start_docstrings( - "Constructs a fast Llama4 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, +class Llama4ImageProcessorKwargs(DefaultFastImageProcessorKwargs): """ - max_patches (`int`, *optional*, defaults to 16): - The maximum number of patches to be extracted from the image. - Can be overridden by the `max_patches` parameter in the `preprocess` method. - resize_to_max_canvas (`bool`, *optional*, defaults to False): - Whether to resize the image to the maximum canvas size. - If True, picks the canvas the allows the largest resizing without distortion. - If False, downsample as little as possible, including no resizing at all, - but never upsample, unless the image is smaller than the patch size. - """, -) + max_patches (`int`, *optional*, defaults to 16): + The maximum number of patches to be extracted from the image. + Can be overridden by the `max_patches` parameter in the `preprocess` method. + resize_to_max_canvas (`bool`, *optional*, defaults to False): + Whether to resize the image to the maximum canvas size. + If True, picks the canvas the allows the largest resizing without distortion. + If False, downsample as little as possible, including no resizing at all, + but never upsample, unless the image is smaller than the patch size. + """ + + max_patches: Optional[int] + resize_to_max_canvas: Optional[bool] + + +@auto_docstring class Llama4ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = [0.5, 0.5, 0.5] @@ -383,19 +379,7 @@ class Llama4ImageProcessorFast(BaseImageProcessorFast): return images - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - max_patches (`int`, *optional*, defaults to 16): - The maximum number of patches to be extracted from the image. - Can be overridden by the `max_patches` parameter in the `preprocess` method. - resize_to_max_canvas (`bool`, *optional*, defaults to False): - Whether to resize the image to the maximum canvas size. - If True, picks the canvas the allows the largest resizing without distortion. - If False, downsample as little as possible, including no resizing at all, - but never upsample, unless the image is smaller than the patch size. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[Llama4ImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/llama4/modeling_llama4.py b/src/transformers/models/llama4/modeling_llama4.py index 00f924b4d7..50a6dc1fc8 100644 --- a/src/transformers/models/llama4/modeling_llama4.py +++ b/src/transformers/models/llama4/modeling_llama4.py @@ -29,24 +29,11 @@ from ...generation import GenerationMixin from ...integrations.hub_kernels import use_kernel_forward_from_hub from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - BaseModelOutput, - BaseModelOutputWithPast, - CausalLMOutputWithPast, - ModelOutput, -) +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, CausalLMOutputWithPast, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_llama4 import Llama4Config, Llama4TextConfig @@ -56,8 +43,6 @@ if is_torch_flex_attn_available(): from ...integrations.flex_attention import make_flex_block_causal_mask logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "meta-ai/Llama-4-17B" -_CONFIG_FOR_DOC = "Llama4Config" class Llama4TextExperts(nn.Module): @@ -445,27 +430,7 @@ class Llama4TextDecoderLayer(nn.Module): return outputs -LLAMA4_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Llama4Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Llama4 Model outputting raw hidden-states without any specific head on top.", - LLAMA4_START_DOCSTRING, -) +@auto_docstring class Llama4PreTrainedModel(PreTrainedModel): config_class = Llama4Config supports_gradient_checkpointing = True @@ -505,85 +470,7 @@ class Llama4PreTrainedModel(PreTrainedModel): module.positional_embedding_vlm.data.normal_(std=module.scale) -LLAMA4_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Llama4 Model outputting raw hidden-states without any specific head on top.", - LLAMA4_START_DOCSTRING, -) +@auto_docstring class Llama4TextModel(Llama4PreTrainedModel): _no_split_modules = ["Llama4TextDecoderLayer"] base_model_prefix = "model" @@ -612,7 +499,7 @@ class Llama4TextModel(Llama4PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA4_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -973,8 +860,7 @@ class Llama4ForCausalLM(Llama4PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(LLAMA4_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -992,20 +878,10 @@ class Llama4ForCausalLM(Llama4PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1165,23 +1041,6 @@ class Llama4VisionPixelShuffleMLP(nn.Module): return self.mlp(encoded_patches) -LLAVA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LlavaConfig`] or [`LlavaVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - # TODO there is a different RoPE for vision encoder, defined as below def reshape_for_broadcast(freqs_ci: torch.Tensor, query: torch.Tensor): ndim = query.ndim @@ -1658,7 +1517,7 @@ class Llama4ForConditionalGeneration(Llama4PreTrainedModel, GenerationMixin): hidden_state = image_outputs.last_hidden_state return hidden_state - @replace_return_docstrings(output_type=Llama4CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1680,20 +1539,10 @@ class Llama4ForConditionalGeneration(Llama4PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Llama4CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/llava/image_processing_llava_fast.py b/src/transformers/models/llava/image_processing_llava_fast.py index d85eb89b7c..e149bf1c62 100644 --- a/src/transformers/models/llava/image_processing_llava_fast.py +++ b/src/transformers/models/llava/image_processing_llava_fast.py @@ -20,8 +20,6 @@ from ...image_processing_utils import ( BatchFeature, ) from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -39,7 +37,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -61,17 +59,16 @@ if is_torchvision_available(): class LlavaFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + Args: + do_pad (`bool`, *optional*): + Whether to pad the image to a square based on the longest edge. + """ + do_pad: Optional[bool] -@add_start_docstrings( - "Constructs a fast Llava image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - do_pad (`bool`, *optional*, defaults to `self.do_pad`): - Whether to pad the image to a square based on the longest edge. Can be overridden by the `do_pad` parameter - """, -) +@auto_docstring class LlavaImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN @@ -90,13 +87,7 @@ class LlavaImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[LlavaFastImageProcessorKwargs]) -> None: super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - do_pad (`bool`, *optional*, defaults to `self.do_pad`): - Whether to pad the image to a square based on the longest edge. Can be overridden by the `do_pad` parameter - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[LlavaFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 42fc86c77c..ebe0e6c599 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -27,26 +27,13 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging from ..auto import AutoModel from .configuration_llava import LlavaConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LlavaConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "llava-hf/llava-1.5-7b-hf" - @dataclass class LlavaModelOutputWithPast(BaseModelOutputWithPast): @@ -143,27 +130,7 @@ class LlavaMultiModalProjector(nn.Module): return hidden_states -LLAVA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LlavaConfig`] or [`LlavaVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Llava Model outputting raw hidden-states without any specific head on top.", - LLAVA_START_DOCSTRING, -) +@auto_docstring class LlavaPreTrainedModel(PreTrainedModel): config_class = LlavaConfig base_model_prefix = "" @@ -191,86 +158,10 @@ class LlavaPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -LLAVA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`CLIPImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Llava model which consists of a vision backbone and a language model, without a language modeling head.""", - LLAVA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Llava model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class LlavaModel(LlavaPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -336,7 +227,7 @@ class LlavaModel(LlavaPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -427,9 +318,10 @@ class LlavaModel(LlavaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The LLAVA model which consists of a vision backbone and a language model.""", - LLAVA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The LLAVA model which consists of a vision backbone and a language model. + """ ) class LlavaForConditionalGeneration(LlavaPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -472,8 +364,7 @@ class LlavaForConditionalGeneration(LlavaPreTrainedModel, GenerationMixin): return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=LlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -495,20 +386,10 @@ class LlavaForConditionalGeneration(LlavaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, LlavaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/llava_next/image_processing_llava_next_fast.py b/src/transformers/models/llava_next/image_processing_llava_next_fast.py index 1118db06e0..8d4b3c48ba 100644 --- a/src/transformers/models/llava_next/image_processing_llava_next_fast.py +++ b/src/transformers/models/llava_next/image_processing_llava_next_fast.py @@ -18,8 +18,6 @@ from typing import List, Optional, Union from ...image_processing_utils import BatchFeature, get_patch_output_size, select_best_resolution from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, divide_to_patches, @@ -39,7 +37,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -57,23 +55,21 @@ if is_torchvision_available(): class LlavaNextFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + image_grid_pinpoints (`List[List[int]]`, *optional*): + A list of possible resolutions to use for processing high resolution images. The best resolution is selected + based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess` + method. + do_pad (`bool`, *optional*): + Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest + number of patches in the batch. Padding will be applied to the bottom and right with zeros. + """ + image_grid_pinpoints: Optional[List[List[int]]] do_pad: Optional[bool] -@add_start_docstrings( - "Constructs a fast ConvNeXT image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - image_grid_pinpoints (`List[List[int]]`, *optional*): - A list of possible resolutions to use for processing high resolution images. The best resolution is selected - based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess` - method. - do_pad (`bool`, *optional*): - Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest - number of patches in the batch. Padding will be applied to the bottom and right with zeros. - """, -) +@auto_docstring class LlavaNextImageProcessorFast(BaseImageProcessorFast): # To be checked against the slow image processor # None values left after checking can be removed @@ -95,17 +91,7 @@ class LlavaNextImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[LlavaNextFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - image_grid_pinpoints (`List`, *optional*): - A list of possible resolutions to use for processing high resolution images. Each item in the list should be a tuple or list - of the form `(height, width)`. - do_pad (`bool`, *optional*): - Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest - number of patches in the batch. Padding will be applied to the bottom and right with zeros. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[LlavaNextFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index ff1cd57d9d..2113224a9d 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -30,23 +30,13 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging from ..auto import AutoModel from .configuration_llava_next import LlavaNextConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LlavaNextConfig" - def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size): """ @@ -251,27 +241,7 @@ class LlavaNextMultiModalProjector(nn.Module): return hidden_states -LLAVA_NEXT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LlavaNextConfig`] or [`LlavaNextVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAVA_NEXT_START_DOCSTRING, -) +@auto_docstring class LlavaNextPreTrainedModel(PreTrainedModel): config_class = LlavaNextConfig base_model_prefix = "" @@ -297,89 +267,10 @@ class LlavaNextPreTrainedModel(PreTrainedModel): module.image_newline.data.normal_(mean=0.0, std=embed_std) -LLAVA_NEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`LlavaNextImageProcessor.__call__`] for details. [`LlavaProcessor`] uses - [`LlavaNextImageProcessor`] for processing images. - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): - The sizes of the images in the batch, being (height, width) for each image. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. - If `"full"`, the full vision features are used. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Llava-Next model which consists of a vision backbone and a language model without language modeling head.""", - LLAVA_NEXT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Llava-Next model which consists of a vision backbone and a language model without language modeling head. + """ ) class LlavaNextModel(LlavaNextPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -533,7 +424,7 @@ class LlavaNextModel(LlavaNextPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_NEXT_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -552,6 +443,12 @@ class LlavaNextModel(LlavaNextPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, LlavaNextModelOutputWithPast]: + r""" + vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. + If `"full"`, the full vision features are used. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -629,9 +526,10 @@ class LlavaNextModel(LlavaNextPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The LLAVA-NeXT model which consists of a vision backbone and a language model.""", - LLAVA_NEXT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The LLAVA-NeXT model which consists of a vision backbone and a language model. + """ ) class LlavaNextForConditionalGeneration(LlavaNextPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -675,8 +573,7 @@ class LlavaNextForConditionalGeneration(LlavaNextPreTrainedModel, GenerationMixi return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_NEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=LlavaNextCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -698,19 +595,14 @@ class LlavaNextForConditionalGeneration(LlavaNextPreTrainedModel, GenerationMixi **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, LlavaNextCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. + If `"full"`, the full vision features are used. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index f124e3b4c1..484030b3e3 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -34,15 +34,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging from ..auto import AutoModel from .configuration_llava_next_video import LlavaNextVideoConfig @@ -50,9 +42,6 @@ from .configuration_llava_next_video import LlavaNextVideoConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LlavaNextVideoConfig" - - @dataclass class LlavaNextVideoModelOutputWithPast(BaseModelOutputWithPast): """ @@ -194,27 +183,7 @@ class LlavaNextVideoMultiModalProjector(nn.Module): return hidden_states -LLAVA_NEXT_VIDEO_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LlavaNextVideoConfig`] or [`LlavaNextVideoVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAVA_NEXT_VIDEO_START_DOCSTRING, -) +@auto_docstring class LlavaNextVideoPreTrainedModel(PreTrainedModel): config_class = LlavaNextVideoConfig base_model_prefix = "" @@ -347,89 +316,10 @@ def unpad_image(tensor, original_size): return unpadded_tensor -LLAVA_NEXT_VIDEO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`LlavaNextVideoImageProcessor.__call__`] for details. [`LlavaProcessor`] uses - [`LlavaNextVideoImageProcessor`] for processing images. - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): - The sizes of the images in the batch, being (height, width) for each image. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. - If `"full"`, the full vision features are used. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Llava-Next model which consists of a vision backbone and a language model without language modeling head.""", - LLAVA_NEXT_VIDEO_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Llava-Next model which consists of a vision backbone and a language model without language modeling head. + """ ) class LlavaNextVideoModel(LlavaNextVideoPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -586,7 +476,7 @@ class LlavaNextVideoModel(LlavaNextVideoPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_NEXT_VIDEO_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -606,6 +496,12 @@ class LlavaNextVideoModel(LlavaNextVideoPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, LlavaNextVideoModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`LlavaNextVideoVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoVideoProcessor`] for processing videos. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -751,9 +647,10 @@ class LlavaNextVideoModel(LlavaNextVideoPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The LLAVA-NeXT model which consists of a vision backbone and a language model.""", - LLAVA_NEXT_VIDEO_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The LLAVA-NeXT model which consists of a vision backbone and a language model. + """ ) class LlavaNextVideoForConditionalGeneration(LlavaNextVideoPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -797,8 +694,7 @@ class LlavaNextVideoForConditionalGeneration(LlavaNextVideoPreTrainedModel, Gene return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_NEXT_VIDEO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=LlavaNextVideoCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -821,22 +717,14 @@ class LlavaNextVideoForConditionalGeneration(LlavaNextVideoPreTrainedModel, Gene **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, LlavaNextVideoCausalLMOutputWithPast]: r""" - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): - The tensors corresponding to the input videos. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`LlavaNextVideoVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses - [`LlavaNextVideoVideoProcessor`] for processing videos. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`LlavaNextVideoVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoVideoProcessor`] for processing videos. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/llava_next_video/modular_llava_next_video.py b/src/transformers/models/llava_next_video/modular_llava_next_video.py index cacf167a86..b00fc91813 100644 --- a/src/transformers/models/llava_next_video/modular_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modular_llava_next_video.py @@ -34,22 +34,13 @@ from transformers.models.llava_next.modeling_llava_next import ( from ...configuration_utils import PretrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import is_torchdynamo_compiling, logging from ..auto import CONFIG_MAPPING, AutoConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LlavaNextVideoConfig" - - class LlavaNextVideoConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`LlavaNextVideoForConditionalGeneration`]. It is used to instantiate an @@ -383,6 +374,12 @@ class LlavaNextVideoModel(LlavaNextModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, LlavaNextVideoModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`LlavaNextVideoVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoVideoProcessor`] for processing videos. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -479,90 +476,7 @@ class LlavaNextVideoModel(LlavaNextModel): ) -LLAVA_NEXT_VIDEO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`LlavaNextVideoImageProcessor.__call__`] for details. [`LlavaProcessor`] uses - [`LlavaNextVideoImageProcessor`] for processing images. - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): - The sizes of the images in the batch, being (height, width) for each image. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. - If `"full"`, the full vision features are used. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class LlavaNextVideoForConditionalGeneration(LlavaNextForConditionalGeneration): - @can_return_tuple - @add_start_docstrings_to_model_forward(LLAVA_NEXT_VIDEO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=LlavaNextVideoCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: torch.LongTensor = None, @@ -585,22 +499,14 @@ class LlavaNextVideoForConditionalGeneration(LlavaNextForConditionalGeneration): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, LlavaNextVideoCausalLMOutputWithPast]: r""" - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): - The tensors corresponding to the input videos. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`LlavaNextVideoVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses - [`LlavaNextVideoVideoProcessor`] for processing videos. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`LlavaNextVideoVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoVideoProcessor`] for processing videos. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py b/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py index df49004eb2..dc7a324c44 100644 --- a/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py +++ b/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py @@ -25,8 +25,6 @@ import torch from ...image_processing_utils import BatchFeature, get_patch_output_size, select_best_resolution from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, divide_to_patches, @@ -44,7 +42,7 @@ from ...image_utils import ( make_flat_list_of_images, ) from ...processing_utils import Unpack -from ...utils import TensorType, add_start_docstrings, is_torchvision_v2_available +from ...utils import TensorType, auto_docstring, is_torchvision_v2_available if is_torchvision_v2_available(): @@ -54,23 +52,21 @@ else: class LlavaOnevisionFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + image_grid_pinpoints (`List[List[int]]`, *optional*): + A list of possible resolutions to use for processing high resolution images. The best resolution is selected + based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess` + method. + do_pad (`bool`, *optional*): + Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest + number of patches in the batch. Padding will be applied to the bottom and right with zeros. + """ + image_grid_pinpoints: Optional[List[List[int]]] do_pad: Optional[bool] -@add_start_docstrings( - "Constructs a fast ConvNeXT image processor. Based on [`SiglipImageProcessor`] with incorporation of processing each video frame.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - image_grid_pinpoints (`List[List[int]]`, *optional*): - A list of possible resolutions to use for processing high resolution images. The best resolution is selected - based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess` - method. Not used for processing videos. - do_pad (`bool`, *optional*): - Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest - number of patches in the batch. Padding will be applied to the bottom and right with zeros. - """, -) +@auto_docstring class LlavaOnevisionImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN @@ -91,17 +87,7 @@ class LlavaOnevisionImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[LlavaOnevisionFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - image_grid_pinpoints (`List`, *optional*): - A list of possible resolutions to use for processing high resolution images. Each item in the list should be a tuple or list - of the form `(height, width)`. - do_pad (`bool`, *optional*): - Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest - number of patches in the batch. Padding will be applied to the bottom and right with zeros. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[LlavaOnevisionFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 73700e2448..a4271297d0 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -36,7 +36,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ( LossKwargs, - add_start_docstrings, + auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging, @@ -189,23 +189,6 @@ class LlavaOnevisionMultiModalProjector(nn.Module): return hidden_states -LLAVA_ONEVISION_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LlavaOnevisionConfig`] or [`LlavaOnevisionVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size): """ Calculate the shape of the image patch grid after the preprocessing for images of any resolution. @@ -313,90 +296,7 @@ def unpad_image(tensor, original_size): return unpadded_tensor -LLAVA_ONEVISION_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`LlavaOnevisionImageProcessor.__call__`] for details. [`LlavaProcessor`] uses - [`LlavaOnevisionImageProcessor`] for processing images. - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): - The sizes of the images in the batch, being (height, width) for each image. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. - If `"full"`, the full vision features are used. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - LLAVA_ONEVISION_START_DOCSTRING, -) +@auto_docstring class LlavaOnevisionPreTrainedModel(PreTrainedModel): config_class = LlavaOnevisionConfig base_model_prefix = "" @@ -422,9 +322,10 @@ class LlavaOnevisionPreTrainedModel(PreTrainedModel): module.image_newline.data.normal_(mean=0.0, std=embed_std) -@add_start_docstrings( - """The Llava-Next model which consists of a vision backbone and a language model without language modeling head.""", - LLAVA_ONEVISION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Llava-Next model which consists of a vision backbone and a language model without language modeling head. + """ ) class LlavaOnevisionModel(LlavaOnevisionPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -575,7 +476,7 @@ class LlavaOnevisionModel(LlavaOnevisionPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings(LLAVA_ONEVISION_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -597,6 +498,21 @@ class LlavaOnevisionModel(LlavaOnevisionPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, LlavaOnevisionModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`LlavaNextVideoProcessor`]. See [`LlavaNextVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoProcessor`] for processing videos. + image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): + The sizes of the videos in the batch, being (height, width) for each frame in the video. + vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. + If `"full"`, the full vision features are used. + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processong image features. The default value is "anyres_max_9". + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -762,9 +678,10 @@ class LlavaOnevisionModel(LlavaOnevisionPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The LLAVA-NeXT model which consists of a vision backbone and a language model.""", - LLAVA_ONEVISION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The LLAVA-NeXT model which consists of a vision backbone and a language model. + """ ) class LlavaOnevisionForConditionalGeneration(LlavaOnevisionPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -808,7 +725,7 @@ class LlavaOnevisionForConditionalGeneration(LlavaOnevisionPreTrainedModel, Gene return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings(LLAVA_ONEVISION_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -833,21 +750,22 @@ class LlavaOnevisionForConditionalGeneration(LlavaOnevisionPreTrainedModel, Gene **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, LlavaOnevisionCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: - [`~LlavaOnevisionCausalLMOutputWithPast`] (if `return_dict=True`) or a `tuple`. + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`LlavaNextVideoProcessor`]. See [`LlavaNextVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoProcessor`] for processing videos. + image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): + The sizes of the videos in the batch, being (height, width) for each frame in the video. + vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. + If `"full"`, the full vision features are used. + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processong image features. The default value is "anyres_max_9". + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/llava_onevision/modular_llava_onevision.py b/src/transformers/models/llava_onevision/modular_llava_onevision.py index c96d2e3619..34e13b5046 100644 --- a/src/transformers/models/llava_onevision/modular_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modular_llava_onevision.py @@ -31,35 +31,15 @@ from transformers.models.llava_next_video.modeling_llava_next_video import ( unpad_image, ) -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING -from ...image_utils import ( - OPENAI_CLIP_MEAN, - OPENAI_CLIP_STD, - PILImageResampling, -) +from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack -from ...utils import add_start_docstrings, can_return_tuple, is_torchdynamo_compiling, logging +from ...utils import auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging logger = logging.get_logger(__name__) -LLAVA_ONEVISION_INPUTS_DOCSTRING = None - -@add_start_docstrings( - "Constructs a fast ConvNeXT image processor. Based on [`SiglipImageProcessor`] with incorporation of processing each video frame.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - image_grid_pinpoints (`List[List[int]]`, *optional*): - A list of possible resolutions to use for processing high resolution images. The best resolution is selected - based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess` - method. Not used for processing videos. - do_pad (`bool`, *optional*): - Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest - number of patches in the batch. Padding will be applied to the bottom and right with zeros. - """, -) class LlavaOnevisionImageProcessorFast(LlavaNextImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN @@ -220,8 +200,6 @@ class LlavaOnevisionModel(LlavaNextVideoModel): return video_features - @can_return_tuple - @add_start_docstrings(LLAVA_ONEVISION_INPUTS_DOCSTRING) def forward( self, input_ids: torch.LongTensor = None, @@ -243,6 +221,21 @@ class LlavaOnevisionModel(LlavaNextVideoModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, LlavaOnevisionModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`LlavaNextVideoProcessor`]. See [`LlavaNextVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoProcessor`] for processing videos. + image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): + The sizes of the videos in the batch, being (height, width) for each frame in the video. + vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. + If `"full"`, the full vision features are used. + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processong image features. The default value is "anyres_max_9". + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -347,7 +340,7 @@ class LlavaOnevisionModel(LlavaNextVideoModel): class LlavaOnevisionForConditionalGeneration(LlavaNextVideoForConditionalGeneration): @can_return_tuple - @add_start_docstrings(LLAVA_ONEVISION_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -372,21 +365,22 @@ class LlavaOnevisionForConditionalGeneration(LlavaNextVideoForConditionalGenerat **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, LlavaOnevisionCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: - [`~LlavaOnevisionCausalLMOutputWithPast`] (if `return_dict=True`) or a `tuple`. + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, frames, num_channels, image_size, image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`LlavaNextVideoProcessor`]. See [`LlavaNextVideoProcessor.__call__`] for details. [`LlavaProcessor`] uses + [`LlavaNextVideoProcessor`] for processing videos. + image_sizes_videos (`torch.LongTensor` of shape `(batch_size, frames, 2)`, *optional*): + The sizes of the videos in the batch, being (height, width) for each frame in the video. + vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. If `"default"`, the CLS token is removed from the vision features. + If `"full"`, the full vision features are used. + vision_aspect_ratio (`str`, *optional*, defaults to `"anyres_max_9"`): + Aspect ratio used when processong image features. The default value is "anyres_max_9". + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/longformer/modeling_longformer.py b/src/transformers/models/longformer/modeling_longformer.py index 9eefa02a66..ea178f3594 100755 --- a/src/transformers/models/longformer/modeling_longformer.py +++ b/src/transformers/models/longformer/modeling_longformer.py @@ -26,22 +26,12 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN, gelu from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_longformer import LongformerConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "allenai/longformer-base-4096" -_CONFIG_FOR_DOC = "LongformerConfig" - @dataclass class LongformerBaseModelOutput(ModelOutput): @@ -1398,12 +1388,8 @@ class LongformerLMHead(nn.Module): self.bias = self.decoder.bias +@auto_docstring class LongformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LongformerConfig base_model_prefix = "longformer" supports_gradient_checkpointing = True @@ -1426,93 +1412,7 @@ class LongformerPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -LONGFORMER_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LongformerConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LONGFORMER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - global_attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to decide the attention given on each token, local attention or global attention. Tokens with global - attention attends to all other tokens, and all other tokens attend to them. This is important for - task-specific finetuning because it makes the model more flexible at representing the task. For example, - for classification, the token should be given global attention. For QA, all question tokens should also - have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more - details. Mask values selected in `[0, 1]`: - - - 0 for local attention (a sliding window attention), - - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). - - head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Longformer Model outputting raw hidden-states without any specific head on top.", - LONGFORMER_START_DOCSTRING, -) +@auto_docstring class LongformerModel(LongformerPreTrainedModel): """ This class copied code from [`RobertaModel`] and overwrote standard self-attention with longformer self-attention @@ -1530,6 +1430,10 @@ class LongformerModel(LongformerPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1625,8 +1529,7 @@ class LongformerModel(LongformerPreTrainedModel): attention_mask = global_attention_mask + 1 return attention_mask - @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=LongformerBaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1641,8 +1544,16 @@ class LongformerModel(LongformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LongformerBaseModelOutputWithPooling]: r""" + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention. Tokens with global + attention attends to all other tokens, and all other tokens attend to them. This is important for + task-specific finetuning because it makes the model more flexible at representing the task. For example, + for classification, the token should be given global attention. For QA, all question tokens should also + have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more + details. Mask values selected in `[0, 1]`: - Returns: + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). Examples: @@ -1677,7 +1588,8 @@ class LongformerModel(LongformerPreTrainedModel): >>> outputs = model(input_ids, attention_mask=attention_mask, global_attention_mask=global_attention_mask) >>> sequence_output = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output - ```""" + ``` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1749,7 +1661,7 @@ class LongformerModel(LongformerPreTrainedModel): ) -@add_start_docstrings("""Longformer Model with a `language modeling` head on top.""", LONGFORMER_START_DOCSTRING) +@auto_docstring class LongformerForMaskedLM(LongformerPreTrainedModel): _tied_weights_keys = ["lm_head.decoder"] @@ -1768,8 +1680,7 @@ class LongformerForMaskedLM(LongformerPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=LongformerMaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1785,16 +1696,22 @@ class LongformerForMaskedLM(LongformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LongformerMaskedLMOutput]: r""" + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention. Tokens with global + attention attends to all other tokens, and all other tokens attend to them. This is important for + task-specific finetuning because it makes the model more flexible at representing the task. For example, + for classification, the token should be given global attention. For QA, all question tokens should also + have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more + details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. - Returns: - - Mask filling example: + Example Mask filling: ```python >>> from transformers import AutoTokenizer, LongformerForMaskedLM @@ -1819,7 +1736,8 @@ class LongformerForMaskedLM(LongformerPreTrainedModel): >>> tokenizer.decode(predictions).split() ['healthy', 'skinny', 'thin', 'good', 'vegetarian'] - ```""" + ``` + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.longformer( @@ -1857,12 +1775,11 @@ class LongformerForMaskedLM(LongformerPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Longformer Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - LONGFORMER_START_DOCSTRING, + """ ) class LongformerForSequenceClassification(LongformerPreTrainedModel): def __init__(self, config): @@ -1876,14 +1793,7 @@ class LongformerForSequenceClassification(LongformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="jpwahle/longformer-base-plagiarism-detection", - output_type=LongformerSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'ORIGINAL'", - expected_loss=5.44, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1899,6 +1809,16 @@ class LongformerForSequenceClassification(LongformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LongformerSequenceClassifierOutput]: r""" + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention. Tokens with global + attention attends to all other tokens, and all other tokens attend to them. This is important for + task-specific finetuning because it makes the model more flexible at representing the task. For example, + for classification, the token should be given global attention. For QA, all question tokens should also + have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more + details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1984,13 +1904,7 @@ class LongformerClassificationHead(nn.Module): return output -@add_start_docstrings( - """ - Longformer Model with a span classification head on top for extractive question-answering tasks like SQuAD / - TriviaQA (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - LONGFORMER_START_DOCSTRING, -) +@auto_docstring class LongformerForQuestionAnswering(LongformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2002,8 +1916,7 @@ class LongformerForQuestionAnswering(LongformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=LongformerQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2020,16 +1933,16 @@ class LongformerForQuestionAnswering(LongformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LongformerQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention. Tokens with global + attention attends to all other tokens, and all other tokens attend to them. This is important for + task-specific finetuning because it makes the model more flexible at representing the task. For example, + for classification, the token should be given global attention. For QA, all question tokens should also + have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more + details. Mask values selected in `[0, 1]`: - Returns: + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). Examples: @@ -2121,13 +2034,7 @@ class LongformerForQuestionAnswering(LongformerPreTrainedModel): ) -@add_start_docstrings( - """ - Longformer Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - LONGFORMER_START_DOCSTRING, -) +@auto_docstring class LongformerForTokenClassification(LongformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2140,17 +2047,7 @@ class LongformerForTokenClassification(LongformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="brad1141/Longformer-finetuned-norm", - output_type=LongformerTokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=( - "['Evidence', 'Evidence', 'Evidence', 'Evidence', 'Evidence', 'Evidence', 'Evidence', 'Evidence'," - " 'Evidence', 'Evidence', 'Evidence', 'Evidence']" - ), - expected_loss=0.63, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2166,6 +2063,16 @@ class LongformerForTokenClassification(LongformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LongformerTokenClassifierOutput]: r""" + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention. Tokens with global + attention attends to all other tokens, and all other tokens attend to them. This is important for + task-specific finetuning because it makes the model more flexible at representing the task. For example, + for classification, the token should be given global attention. For QA, all question tokens should also + have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more + details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -2209,13 +2116,7 @@ class LongformerForTokenClassification(LongformerPreTrainedModel): ) -@add_start_docstrings( - """ - Longformer Model with a multiple choice classification head on top (a linear layer on top of the pooled output and - a softmax) e.g. for RocStories/SWAG tasks. - """, - LONGFORMER_START_DOCSTRING, -) +@auto_docstring class LongformerForMultipleChoice(LongformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2227,14 +2128,7 @@ class LongformerForMultipleChoice(LongformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - LONGFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LongformerMultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2250,10 +2144,44 @@ class LongformerForMultipleChoice(LongformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LongformerMultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + global_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Mask to decide the attention given on each token, local attention or global attention. Tokens with global + attention attends to all other tokens, and all other tokens attend to them. This is important for + task-specific finetuning because it makes the model more flexible at representing the task. For example, + for classification, the token should be given global attention. For QA, all question tokens should also + have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more + details. Mask values selected in `[0, 1]`: + + - 0 for local attention (a sliding window attention), + - 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them). labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/longt5/modeling_longt5.py b/src/transformers/models/longt5/modeling_longt5.py index ea9a046a88..7cb9aa8e94 100644 --- a/src/transformers/models/longt5/modeling_longt5.py +++ b/src/transformers/models/longt5/modeling_longt5.py @@ -38,13 +38,11 @@ from ...pytorch_utils import ALL_LAYERNORM_LAYERS, find_pruneable_heads_and_indi from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from .configuration_longt5 import LongT5Config @@ -57,8 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LongT5Config" -_CHECKPOINT_FOR_DOC = "google/long-t5-local-base" # TODO: Update before the merge @@ -1249,12 +1245,8 @@ class LongT5Block(nn.Module): return outputs # hidden-states, present_key_value_states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights) +@auto_docstring class LongT5PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LongT5Config base_model_prefix = "transformer" supports_gradient_checkpointing = True @@ -1723,164 +1715,6 @@ class LongT5Stack(LongT5PreTrainedModel): return causal_mask -LONGT5_START_DOCSTRING = r""" - - The LongT5 model was proposed in [LongT5: Efficient Text-To-Text Transformer for Long - Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo - Ni, Yun-Hsuan Sung and Yinfei Yang. It's an encoder-decoder transformer pre-trained in a text-to-text denoising - generative setting. LongT5 model is an extension of T5 model, and it enables using one of the two different - efficient attention mechanisms - (1) Local attention, or (2) Transient-Global attention. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LongT5Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LONGT5_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so - you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - [What are input IDs?](../glossary#input-ids) - - To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 - Training](./longt5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5 - Training](./longt5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - -LONGT5_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so - you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 - Training](./longt5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask __HEAD_MASK_WARNING_MSG = """ The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently, @@ -1890,10 +1724,7 @@ num_heads)`. """ -@add_start_docstrings( - "The bare LONGT5 Model transformer outputting raw hidden-states without any specific head on top.", - LONGT5_START_DOCSTRING, -) +@auto_docstring class LongT5Model(LongT5PreTrainedModel): _keys_to_ignore_on_load_unexpected = [ r"decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight", @@ -1946,8 +1777,7 @@ class LongT5Model(LongT5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(LONGT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1968,7 +1798,46 @@ class LongT5Model(LongT5PreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 + Training](./longt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5 + Training](./longt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -2050,7 +1919,11 @@ class LongT5Model(LongT5PreTrainedModel): ) -@add_start_docstrings("""LONGT5 Model with a `language modeling` head on top.""", LONGT5_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + LONGT5 Model with a `language modeling` head on top. + """ +) class LongT5ForConditionalGeneration(LongT5PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_unexpected = [ r"decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight", @@ -2105,8 +1978,7 @@ class LongT5ForConditionalGeneration(LongT5PreTrainedModel, GenerationMixin): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(LONGT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2128,13 +2000,51 @@ class LongT5ForConditionalGeneration(LongT5PreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 + Training](./longt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5 + Training](./longt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -2265,10 +2175,7 @@ class LongT5ForConditionalGeneration(LongT5PreTrainedModel, GenerationMixin): return reordered_decoder_past -@add_start_docstrings( - "The bare LONGT5 Model transformer outputting encoder's raw hidden-states without any specific head on top.", - LONGT5_START_DOCSTRING, -) +@auto_docstring class LongT5EncoderModel(LongT5PreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight"] _keys_to_ignore_on_load_unexpected = [r"decoder"] @@ -2307,8 +2214,7 @@ class LongT5EncoderModel(LongT5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(LONGT5_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2320,7 +2226,15 @@ class LongT5EncoderModel(LongT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5 + Training](./longt5#training). Example: diff --git a/src/transformers/models/luke/modeling_luke.py b/src/transformers/models/luke/modeling_luke.py index 7bc27bcd9c..b921dbda67 100644 --- a/src/transformers/models/luke/modeling_luke.py +++ b/src/transformers/models/luke/modeling_luke.py @@ -27,22 +27,12 @@ from ...activations import ACT2FN, gelu from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_luke import LukeConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "LukeConfig" -_CHECKPOINT_FOR_DOC = "studio-ousia/luke-base" - @dataclass class BaseLukeModelOutputWithPooling(BaseModelOutputWithPooling): @@ -883,12 +873,8 @@ class EntityPredictionHead(nn.Module): return hidden_states +@auto_docstring class LukePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LukeConfig base_model_prefix = "luke" supports_gradient_checkpointing = True @@ -912,104 +898,17 @@ class LukePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -LUKE_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LukeConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LUKE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - - entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): - Indices of entity tokens in the entity vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): - Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: - - - 1 for entity tokens that are **not masked**, - - 0 for entity tokens that are **masked**. - - entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): - Segment token indices to indicate first and second portions of the entity token inputs. Indices are - selected in `[0, 1]`: - - - 0 corresponds to a *portion A* entity token, - - 1 corresponds to a *portion B* entity token. - - entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): - Indices of positions of each input entity in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare LUKE model transformer outputting raw hidden-states for both word tokens and entities without any" - " specific head on top.", - LUKE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare LUKE model transformer outputting raw hidden-states for both word tokens and entities without any + """ ) class LukeModel(LukePreTrainedModel): def __init__(self, config: LukeConfig, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -1037,8 +936,7 @@ class LukeModel(LukePreTrainedModel): def _prune_heads(self, heads_to_prune): raise NotImplementedError("LUKE does not support the pruning of attention heads") - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseLukeModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1056,8 +954,25 @@ class LukeModel(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseLukeModelOutputWithPooling]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. - Returns: + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. Examples: @@ -1256,12 +1171,11 @@ class LukeLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LUKE model with a language modeling head and entity prediction head on top for masked language modeling and masked entity prediction. - """, - LUKE_START_DOCSTRING, + """ ) class LukeForMaskedLM(LukePreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias", "entity_predictions.decoder.weight"] @@ -1289,8 +1203,7 @@ class LukeForMaskedLM(LukePreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=LukeMaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1310,6 +1223,25 @@ class LukeForMaskedLM(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LukeMaskedLMOutput]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the @@ -1318,9 +1250,6 @@ class LukeForMaskedLM(LukePreTrainedModel): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - - Returns: - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1391,12 +1320,11 @@ class LukeForMaskedLM(LukePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LUKE model with a classification head on top (a linear layer on top of the hidden state of the first entity token) for entity classification tasks, such as Open Entity. - """, - LUKE_START_DOCSTRING, + """ ) class LukeForEntityClassification(LukePreTrainedModel): def __init__(self, config): @@ -1411,8 +1339,7 @@ class LukeForEntityClassification(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=EntityClassificationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1431,6 +1358,25 @@ class LukeForEntityClassification(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, EntityClassificationOutput]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. labels (`torch.LongTensor` of shape `(batch_size,)` or `(batch_size, num_labels)`, *optional*): Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss is used for the single-label classification. In this case, labels should contain the indices that should be in @@ -1438,8 +1384,6 @@ class LukeForEntityClassification(LukePreTrainedModel): loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0 and 1 indicate false and true, respectively. - Returns: - Examples: ```python @@ -1506,12 +1450,11 @@ class LukeForEntityClassification(LukePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LUKE model with a classification head on top (a linear layer on top of the hidden states of the two entity tokens) for entity pair classification tasks, such as TACRED. - """, - LUKE_START_DOCSTRING, + """ ) class LukeForEntityPairClassification(LukePreTrainedModel): def __init__(self, config): @@ -1526,8 +1469,7 @@ class LukeForEntityPairClassification(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=EntityPairClassificationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1546,6 +1488,25 @@ class LukeForEntityPairClassification(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, EntityPairClassificationOutput]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. labels (`torch.LongTensor` of shape `(batch_size,)` or `(batch_size, num_labels)`, *optional*): Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss is used for the single-label classification. In this case, labels should contain the indices that should be in @@ -1553,8 +1514,6 @@ class LukeForEntityPairClassification(LukePreTrainedModel): loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0 and 1 indicate false and true, respectively. - Returns: - Examples: ```python @@ -1626,12 +1585,11 @@ class LukeForEntityPairClassification(LukePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LUKE model with a span classification head on top (a linear layer on top of the hidden states output) for tasks such as named entity recognition. - """, - LUKE_START_DOCSTRING, + """ ) class LukeForEntitySpanClassification(LukePreTrainedModel): def __init__(self, config): @@ -1646,8 +1604,7 @@ class LukeForEntitySpanClassification(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=EntitySpanClassificationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1668,12 +1625,29 @@ class LukeForEntitySpanClassification(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, EntitySpanClassificationOutput]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. entity_start_positions (`torch.LongTensor`): The start positions of entities in the word token sequence. - entity_end_positions (`torch.LongTensor`): The end positions of entities in the word token sequence. - labels (`torch.LongTensor` of shape `(batch_size, entity_length)` or `(batch_size, entity_length, num_labels)`, *optional*): Labels for computing the classification loss. If the shape is `(batch_size, entity_length)`, the cross entropy loss is used for the single-label classification. In this case, labels should contain the indices @@ -1681,8 +1655,6 @@ class LukeForEntitySpanClassification(LukePreTrainedModel): num_labels)`, the binary cross entropy loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0 and 1 indicate false and true, respectively. - Returns: - Examples: ```python @@ -1772,12 +1744,11 @@ class LukeForEntitySpanClassification(LukePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LUKE Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - LUKE_START_DOCSTRING, + """ ) class LukeForSequenceClassification(LukePreTrainedModel): def __init__(self, config): @@ -1792,12 +1763,7 @@ class LukeForSequenceClassification(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LukeSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1816,6 +1782,25 @@ class LukeForSequenceClassification(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LukeSequenceClassifierOutput]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1885,13 +1870,12 @@ class LukeForSequenceClassification(LukePreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The LUKE Model with a token classification head on top (a linear layer on top of the hidden-states output). To solve Named-Entity Recognition (NER) task using LUKE, `LukeForEntitySpanClassification` is more suitable than this class. - """, - LUKE_START_DOCSTRING, + """ ) class LukeForTokenClassification(LukePreTrainedModel): def __init__(self, config): @@ -1907,12 +1891,7 @@ class LukeForTokenClassification(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LukeTokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1931,6 +1910,25 @@ class LukeForTokenClassification(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LukeTokenClassifierOutput]: r""" + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1982,13 +1980,7 @@ class LukeForTokenClassification(LukePreTrainedModel): ) -@add_start_docstrings( - """ - The LUKE Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - LUKE_START_DOCSTRING, -) +@auto_docstring class LukeForQuestionAnswering(LukePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2001,12 +1993,7 @@ class LukeForQuestionAnswering(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LukeQuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2026,14 +2013,25 @@ class LukeForQuestionAnswering(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LukeQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2101,13 +2099,7 @@ class LukeForQuestionAnswering(LukePreTrainedModel): ) -@add_start_docstrings( - """ - The LUKE Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - LUKE_START_DOCSTRING, -) +@auto_docstring class LukeForMultipleChoice(LukePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2121,12 +2113,7 @@ class LukeForMultipleChoice(LukePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(LUKE_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LukeMultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2145,6 +2132,49 @@ class LukeForMultipleChoice(LukePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, LukeMultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`): + Indices of entity tokens in the entity vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*): + Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`: + + - 1 for entity tokens that are **not masked**, + - 0 for entity tokens that are **masked**. + entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*): + Segment token indices to indicate first and second portions of the entity token inputs. Indices are + selected in `[0, 1]`: + + - 0 corresponds to a *portion A* entity token, + - 1 corresponds to a *portion B* entity token. + entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*): + Indices of positions of each input entity in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See diff --git a/src/transformers/models/lxmert/modeling_lxmert.py b/src/transformers/models/lxmert/modeling_lxmert.py index 4f25a67177..8e73378df5 100644 --- a/src/transformers/models/lxmert/modeling_lxmert.py +++ b/src/transformers/models/lxmert/modeling_lxmert.py @@ -26,22 +26,12 @@ from torch.nn import CrossEntropyLoss, SmoothL1Loss from ...activations import ACT2FN, gelu from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_lxmert import LxmertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "unc-nlp/lxmert-base-uncased" -_CONFIG_FOR_DOC = "LxmertConfig" - class GeLU(nn.Module): def __init__(self): @@ -764,12 +754,8 @@ class LxmertPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class LxmertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = LxmertConfig load_tf_weights = load_tf_weights_in_lxmert base_model_prefix = "lxmert" @@ -794,90 +780,7 @@ class LxmertPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -LXMERT_START_DOCSTRING = r""" - - The LXMERT model was proposed in [LXMERT: Learning Cross-Modality Encoder Representations from - Transformers](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal. It's a vision and language transformer - model, pretrained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MSCOCO captions, and Visual - genome, using a combination of masked language modeling, region of interest feature regression, cross entropy loss - for question answering attribute prediction, and object tag prediction. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`LxmertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -LXMERT_INPUTS_DOCSTRING = r""" - - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`): - This input represents visual features. They ROI pooled object features from bounding boxes using a - faster-RCNN model) - - These are currently not provided by the transformers library. - visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`): - This input represents spatial features corresponding to their relative (via index) visual features. The - pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to - 1. - - These are currently not provided by the transformers library. - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - visual_attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Lxmert Model transformer outputting raw hidden-states without any specific head on top.", - LXMERT_START_DOCSTRING, -) +@auto_docstring class LxmertModel(LxmertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -893,12 +796,7 @@ class LxmertModel(LxmertPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.embeddings.word_embeddings = new_embeddings - @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LxmertModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -912,6 +810,26 @@ class LxmertModel(LxmertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[LxmertModelOutput, Tuple[torch.FloatTensor]]: + r""" + visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`): + This input represents visual features. They ROI pooled object features from bounding boxes using a + faster-RCNN model) + + These are currently not provided by the transformers library. + visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`): + This input represents spatial features corresponding to their relative (via index) visual features. The + pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to + 1. + + These are currently not provided by the transformers library. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1012,10 +930,7 @@ class LxmertModel(LxmertPreTrainedModel): ) -@add_start_docstrings( - """Lxmert Model with a specified pretraining head on top.""", - LXMERT_START_DOCSTRING, -) +@auto_docstring class LxmertForPreTraining(LxmertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight"] @@ -1166,8 +1081,7 @@ class LxmertForPreTraining(LxmertPreTrainedModel): return new_qa_logit_layer - @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=LxmertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1187,6 +1101,24 @@ class LxmertForPreTraining(LxmertPreTrainedModel): **kwargs, ) -> Union[LxmertForPreTrainingOutput, Tuple[torch.FloatTensor]]: r""" + visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`): + This input represents visual features. They ROI pooled object features from bounding boxes using a + faster-RCNN model) + + These are currently not provided by the transformers library. + visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`): + This input represents spatial features corresponding to their relative (via index) visual features. The + pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to + 1. + + These are currently not provided by the transformers library. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the @@ -1203,8 +1135,6 @@ class LxmertForPreTraining(LxmertPreTrainedModel): - 1 indicates that the sentence does match the image. ans (`Torch.Tensor` of shape `(batch_size)`, *optional*): a one hot representation hof the correct answer *optional* - - Returns: """ if "masked_lm_labels" in kwargs: @@ -1301,9 +1231,10 @@ class LxmertForPreTraining(LxmertPreTrainedModel): ) -@add_start_docstrings( - """Lxmert Model with a visual-answering head on top for downstream QA tasks""", - LXMERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Lxmert Model with a visual-answering head on top for downstream QA tasks + """ ) class LxmertForQuestionAnswering(LxmertPreTrainedModel): def __init__(self, config): @@ -1397,12 +1328,7 @@ class LxmertForQuestionAnswering(LxmertPreTrainedModel): return new_qa_logit_layer - @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=LxmertForQuestionAnsweringOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1418,6 +1344,24 @@ class LxmertForQuestionAnswering(LxmertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[LxmertForQuestionAnsweringOutput, Tuple[torch.FloatTensor]]: r""" + visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`): + This input represents visual features. They ROI pooled object features from bounding boxes using a + faster-RCNN model) + + These are currently not provided by the transformers library. + visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`): + This input represents spatial features corresponding to their relative (via index) visual features. The + pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to + 1. + + These are currently not provided by the transformers library. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) labels (`Torch.Tensor` of shape `(batch_size)`, *optional*): A one-hot representation of the correct answer """ diff --git a/src/transformers/models/m2m_100/modeling_m2m_100.py b/src/transformers/models/m2m_100/modeling_m2m_100.py index c4cbc21922..3e01afd432 100755 --- a/src/transformers/models/m2m_100/modeling_m2m_100.py +++ b/src/transformers/models/m2m_100/modeling_m2m_100.py @@ -39,14 +39,7 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_m2m_100 import M2M100Config @@ -56,9 +49,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "M2M100Config" -_CHECKPOINT_FOR_DOC = "facebook/m2m100_418M" - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -773,6 +763,7 @@ class M2M100DecoderLayer(nn.Module): return outputs +@auto_docstring class M2M100PreTrainedModel(PreTrainedModel): config_class = M2M100Config base_model_prefix = "model" @@ -793,130 +784,6 @@ class M2M100PreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -M2M_100_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`M2M100Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -M2M_100_GENERATION_EXAMPLE = r""" - Translation example: - - ```python - >>> from transformers import AutoTokenizer, M2M100ForConditionalGeneration - - >>> model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M") - - >>> text_to_translate = "Life is like a box of chocolates" - >>> model_inputs = tokenizer(text_to_translate, return_tensors="pt") - - >>> # translate to French - >>> gen_tokens = model.generate(**model_inputs, forced_bos_token_id=tokenizer.get_lang_id("fr")) - >>> print(tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)) - ``` -""" - -M2M_100_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - M2M100 uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class M2M100Encoder(M2M100PreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1389,10 +1256,7 @@ class M2M100Decoder(M2M100PreTrainedModel): ) -@add_start_docstrings( - "The bare M2M100 Model outputting raw hidden-states without any specific head on top.", - M2M_100_START_DOCSTRING, -) +@auto_docstring class M2M100Model(M2M100PreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1433,12 +1297,7 @@ class M2M100Model(M2M100PreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(M2M_100_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1457,6 +1316,28 @@ class M2M100Model(M2M100PreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + M2M100 uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1513,8 +1394,10 @@ class M2M100Model(M2M100PreTrainedModel): ) -@add_start_docstrings( - "The M2M100 Model with a language modeling head. Can be used for summarization.", M2M_100_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The M2M100 Model with a language modeling head. Can be used for summarization. + """ ) class M2M100ForConditionalGeneration(M2M100PreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1540,9 +1423,7 @@ class M2M100ForConditionalGeneration(M2M100PreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(M2M_100_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(M2M_100_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1563,12 +1444,46 @@ class M2M100ForConditionalGeneration(M2M100PreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + M2M100 uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Translation: + + ```python + >>> from transformers import AutoTokenizer, M2M100ForConditionalGeneration + + >>> model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M") + + >>> text_to_translate = "Life is like a box of chocolates" + >>> model_inputs = tokenizer(text_to_translate, return_tensors="pt") + + >>> # translate to French + >>> gen_tokens = model.generate(**model_inputs, forced_bos_token_id=tokenizer.get_lang_id("fr")) + >>> print(tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)) + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/mamba/modeling_mamba.py b/src/transformers/models/mamba/modeling_mamba.py index 7fe46b1164..7f2497849f 100644 --- a/src/transformers/models/mamba/modeling_mamba.py +++ b/src/transformers/models/mamba/modeling_mamba.py @@ -29,9 +29,7 @@ from ...generation import GenerationMixin from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) from ...utils.import_utils import is_causal_conv1d_available, is_mamba_ssm_available, is_mambapy_available @@ -60,9 +58,6 @@ is_fast_path_available = all( (selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn) ) -_CHECKPOINT_FOR_DOC = "state-spaces/mamba-130m-hf" -_CONFIG_FOR_DOC = "MambaConfig" - class MambaMixer(nn.Module): """ @@ -376,12 +371,8 @@ class MambaBlock(nn.Module): return hidden_states +@auto_docstring class MambaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MambaConfig base_model_prefix = "backbone" _no_split_modules = ["MambaBlock", "MambaMixer"] @@ -489,59 +480,7 @@ class MambaCausalLMOutput(ModelOutput): hidden_states: Optional[Tuple[torch.FloatTensor]] = None -MAMBA_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MambaConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MAMBA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - Indices of input sequence tokens in the vocabulary. - - If `cache_params.seqlen_offset>0`, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - cache_params (`MambaCache`, *optional*): - If passed along, the model uses the previous state in all the blocks (which will give the output for the - `input_ids` provided as if the model add `state_input_ids + input_ids` as context). - use_cache (`bool`, *optional*): - If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare MAMBA Model transformer outputting raw hidden-states without any specific head on top.", - MAMBA_START_DOCSTRING, -) +@auto_docstring class MambaModel(MambaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -567,12 +506,7 @@ class MambaModel(MambaPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.embeddings = new_embeddings - @add_start_docstrings_to_model_forward(MAMBA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MambaOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -584,6 +518,13 @@ class MambaModel(MambaPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, ) -> Union[Tuple, MambaOutput]: + r""" + cache_params (`MambaCache`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). + use_cache (`bool`, *optional*): + If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. + """ output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -650,12 +591,11 @@ class MambaModel(MambaPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The MAMBA Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - MAMBA_START_DOCSTRING, + """ ) class MambaForCausalLM(MambaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -746,12 +686,7 @@ class MambaForCausalLM(MambaPreTrainedModel, GenerationMixin): ) return model_inputs - @add_start_docstrings_to_model_forward(MAMBA_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MambaCausalLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -766,10 +701,15 @@ class MambaForCausalLM(MambaPreTrainedModel, GenerationMixin): **kwargs, # for now we need this for generation ) -> Union[Tuple, MambaCausalLMOutput]: r""" + cache_params (`MambaCache`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + use_cache (`bool`, *optional*): + If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/mamba2/modeling_mamba2.py b/src/transformers/models/mamba2/modeling_mamba2.py index 99f7ebbb88..f097aed986 100644 --- a/src/transformers/models/mamba2/modeling_mamba2.py +++ b/src/transformers/models/mamba2/modeling_mamba2.py @@ -28,9 +28,7 @@ from ...generation import GenerationMixin from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) from ...utils.import_utils import is_causal_conv1d_available, is_mamba_2_ssm_available @@ -61,9 +59,6 @@ is_fast_path_available = all( ) ) -_CHECKPOINT_FOR_DOC = "mistralai/mamba-codestral-7B-v0.1" -_CONFIG_FOR_DOC = "Mamba2Config" - # Helper methods for segment sum computation @@ -715,12 +710,8 @@ class Mamba2Block(nn.Module): return hidden_states +@auto_docstring class Mamba2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Mamba2Config base_model_prefix = "backbone" _no_split_modules = ["Mamba2Block"] @@ -825,65 +816,7 @@ class Mamba2CausalLMOutput(ModelOutput): hidden_states: Optional[Tuple[torch.FloatTensor]] = None -MAMBA2_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Mamba2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MAMBA2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - Indices of input sequence tokens in the vocabulary. - - If `cache_params.seqlen_offset>0`, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - cache_params (`Mamba2Cache`, *optional*): - If passed along, the model uses the previous state in all the blocks (which will give the output for the - `input_ids` provided as if the model add `state_input_ids + input_ids` as context). - use_cache (`bool`, *optional*): - If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - The position of the current input in the cache. This is used to ensure that the cache is correctly updated. - If `cache_params` is passed, `cache_position` should also be passed. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) -""" - - -@add_start_docstrings( - "The bare MAMBA2 Model transformer outputting raw hidden-states without any specific head on top.", - MAMBA2_START_DOCSTRING, -) +@auto_docstring class Mamba2Model(Mamba2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -909,12 +842,7 @@ class Mamba2Model(Mamba2PreTrainedModel): def set_input_embeddings(self, new_embeddings): self.embeddings = new_embeddings - @add_start_docstrings_to_model_forward(MAMBA2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Mamba2Output, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -927,6 +855,16 @@ class Mamba2Model(Mamba2PreTrainedModel): attention_mask: Optional[torch.Tensor] = None, **kwargs, ) -> Union[Tuple, Mamba2Output]: + r""" + cache_params (`Mamba2Cache`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). + use_cache (`bool`, *optional*): + If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. + cache_position (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + The position of the current input in the cache. This is used to ensure that the cache is correctly updated. + If `cache_params` is passed, `cache_position` should also be passed. + """ output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -993,12 +931,11 @@ class Mamba2Model(Mamba2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The MAMBA2 Model transformer with a language modeling head on top (linear layer with weights not tied to the input embeddings). - """, - MAMBA2_START_DOCSTRING, + """ ) class Mamba2ForCausalLM(Mamba2PreTrainedModel, GenerationMixin): _tied_weights_keys = [] @@ -1069,12 +1006,7 @@ class Mamba2ForCausalLM(Mamba2PreTrainedModel, GenerationMixin): ) return model_inputs - @add_start_docstrings_to_model_forward(MAMBA2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Mamba2CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1089,10 +1021,18 @@ class Mamba2ForCausalLM(Mamba2PreTrainedModel, GenerationMixin): **kwargs, # for now we need this for generation ) -> Union[Tuple, Mamba2CausalLMOutput]: r""" + cache_params (`Mamba2Cache`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + use_cache (`bool`, *optional*): + If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. + cache_position (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + The position of the current input in the cache. This is used to ensure that the cache is correctly updated. + If `cache_params` is passed, `cache_position` should also be passed. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/marian/modeling_marian.py b/src/transformers/models/marian/modeling_marian.py index 5eeab498a5..f544641e0d 100755 --- a/src/transformers/models/marian/modeling_marian.py +++ b/src/transformers/models/marian/modeling_marian.py @@ -35,21 +35,12 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_marian import MarianConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MarianConfig" -_CHECKPOINT_FOR_DOC = "Helsinki-NLP/opus-mt-en-de" - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -453,6 +444,7 @@ class MarianDecoderLayer(nn.Module): return outputs +@auto_docstring class MarianPreTrainedModel(PreTrainedModel): config_class = MarianConfig base_model_prefix = "model" @@ -483,138 +475,6 @@ class MarianPreTrainedModel(PreTrainedModel): return dummy_inputs -MARIAN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MarianConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MARIAN_GENERATION_EXAMPLE = r""" - Pytorch version of marian-nmt's transformer.h (c++). Designed for the OPUS-NMT translation checkpoints. Available - models are listed [here](https://huggingface.co/models?search=Helsinki-NLP). - - Examples: - - ```python - >>> from transformers import AutoTokenizer, MarianMTModel - - >>> src = "fr" # source language - >>> trg = "en" # target language - - >>> model_name = f"Helsinki-NLP/opus-mt-{src}-{trg}" - >>> model = MarianMTModel.from_pretrained(model_name) - >>> tokenizer = AutoTokenizer.from_pretrained(model_name) - - >>> sample_text = "où est l'arrêt de bus ?" - >>> batch = tokenizer([sample_text], return_tensors="pt") - - >>> generated_ids = model.generate(**batch) - >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] - "Where's the bus stop?" - ``` -""" - -MARIAN_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Marian uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class MarianEncoder(MarianPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1033,9 +893,7 @@ class MarianDecoder(MarianPreTrainedModel): ) -@add_start_docstrings( - "The bare Marian Model outputting raw hidden-states without any specific head on top.", MARIAN_START_DOCSTRING -) +@auto_docstring class MarianModel(MarianPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1120,8 +978,7 @@ class MarianModel(MarianPreTrainedModel): return model_embeds - @add_start_docstrings_to_model_forward(MARIAN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1141,7 +998,26 @@ class MarianModel(MarianPreTrainedModel): return_dict: Optional[bool] = None, ) -> Seq2SeqModelOutput: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Marian uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1219,8 +1095,10 @@ class MarianModel(MarianPreTrainedModel): ) -@add_start_docstrings( - "The Marian Model with a language modeling head. Can be used for summarization.", MARIAN_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The Marian Model with a language modeling head. Can be used for summarization. + """ ) class MarianMTModel(MarianPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1355,9 +1233,7 @@ class MarianMTModel(MarianPreTrainedModel, GenerationMixin): if hasattr(module, "_tie_weights"): module._tie_weights() - @add_start_docstrings_to_model_forward(MARIAN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(MARIAN_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1378,13 +1254,50 @@ class MarianMTModel(MarianPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Seq2SeqLMOutput: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Marian uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example: + ```python + >>> from transformers import AutoTokenizer, MarianMTModel + + >>> src = "fr" # source language + >>> trg = "en" # target language + + >>> model_name = f"Helsinki-NLP/opus-mt-{src}-{trg}" + >>> model = MarianMTModel.from_pretrained(model_name) + >>> tokenizer = AutoTokenizer.from_pretrained(model_name) + + >>> sample_text = "où est l'arrêt de bus ?" + >>> batch = tokenizer([sample_text], return_tensors="pt") + + >>> generated_ids = model.generate(**batch) + >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] + "Where's the bus stop?" + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1501,7 +1414,7 @@ class MarianForCausalLM(MarianPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1519,72 +1432,15 @@ class MarianForCausalLM(MarianPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/markuplm/modeling_markuplm.py b/src/transformers/models/markuplm/modeling_markuplm.py index f47483d9d8..f2ebe388de 100755 --- a/src/transformers/models/markuplm/modeling_markuplm.py +++ b/src/transformers/models/markuplm/modeling_markuplm.py @@ -24,11 +24,6 @@ from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...file_utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -43,15 +38,12 @@ from ...modeling_utils import ( find_pruneable_heads_and_indices, prune_linear_layer, ) -from ...utils import logging +from ...utils import auto_docstring, logging from .configuration_markuplm import MarkupLMConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/markuplm-base" -_CONFIG_FOR_DOC = "MarkupLMConfig" - class XPathEmbeddings(nn.Module): """Construct the embeddings from xpath tags and subscripts. @@ -706,12 +698,8 @@ class MarkupLMEncoder(nn.Module): ) +@auto_docstring class MarkupLMPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MarkupLMConfig base_model_prefix = "markuplm" @@ -741,73 +729,14 @@ class MarkupLMPreTrainedModel(PreTrainedModel): ) -MARKUPLM_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MarkupLMConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MARKUPLM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - xpath_tags_seq (`torch.LongTensor` of shape `({0}, config.max_depth)`, *optional*): - Tag IDs for each token in the input sequence, padded up to config.max_depth. - - xpath_subs_seq (`torch.LongTensor` of shape `({0}, config.max_depth)`, *optional*): - Subscript IDs for each token in the input sequence, padded up to config.max_depth. - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for - tokens that are NOT MASKED, `0` for MASKED tokens. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1` - indicates the head is **not masked**, `0` indicates the head is **masked**. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - If set to `True`, the model will return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MarkupLM Model transformer outputting raw hidden-states without any specific head on top.", - MARKUPLM_START_DOCSTRING, -) +@auto_docstring class MarkupLMModel(MarkupLMPreTrainedModel): # Copied from transformers.models.clap.modeling_clap.ClapTextModel.__init__ with ClapText->MarkupLM def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -833,8 +762,7 @@ class MarkupLMModel(MarkupLMPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MARKUPLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPoolingAndCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -850,7 +778,10 @@ class MarkupLMModel(MarkupLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: r""" - Returns: + xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Tag IDs for each token in the input sequence, padded up to config.max_depth. + xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Subscript IDs for each token in the input sequence, padded up to config.max_depth. Examples: @@ -948,13 +879,7 @@ class MarkupLMModel(MarkupLMPreTrainedModel): return reordered_past -@add_start_docstrings( - """ - MarkupLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MARKUPLM_START_DOCSTRING, -) +@auto_docstring class MarkupLMForQuestionAnswering(MarkupLMPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForQuestionAnswering.__init__ with bert->markuplm, Bert->MarkupLM def __init__(self, config): @@ -967,8 +892,7 @@ class MarkupLMForQuestionAnswering(MarkupLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MARKUPLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -986,16 +910,10 @@ class MarkupLMForQuestionAnswering(MarkupLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - - Returns: + xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Tag IDs for each token in the input sequence, padded up to config.max_depth. + xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Subscript IDs for each token in the input sequence, padded up to config.max_depth. Examples: @@ -1074,7 +992,11 @@ class MarkupLMForQuestionAnswering(MarkupLMPreTrainedModel): ) -@add_start_docstrings("""MarkupLM Model with a `token_classification` head on top.""", MARKUPLM_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + MarkupLM Model with a `token_classification` head on top. + """ +) class MarkupLMForTokenClassification(MarkupLMPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForTokenClassification.__init__ with bert->markuplm, Bert->MarkupLM def __init__(self, config): @@ -1091,8 +1013,7 @@ class MarkupLMForTokenClassification(MarkupLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MARKUPLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1109,11 +1030,13 @@ class MarkupLMForTokenClassification(MarkupLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Tag IDs for each token in the input sequence, padded up to config.max_depth. + xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Subscript IDs for each token in the input sequence, padded up to config.max_depth. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Examples: ```python @@ -1174,12 +1097,11 @@ class MarkupLMForTokenClassification(MarkupLMPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MarkupLM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MARKUPLM_START_DOCSTRING, + """ ) class MarkupLMForSequenceClassification(MarkupLMPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForSequenceClassification.__init__ with bert->markuplm, Bert->MarkupLM @@ -1198,8 +1120,7 @@ class MarkupLMForSequenceClassification(MarkupLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MARKUPLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1216,13 +1137,15 @@ class MarkupLMForSequenceClassification(MarkupLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Tag IDs for each token in the input sequence, padded up to config.max_depth. + xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): + Subscript IDs for each token in the input sequence, padded up to config.max_depth. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/mask2former/modeling_mask2former.py b/src/transformers/models/mask2former/modeling_mask2former.py index 551ef08588..7650cfcd0f 100644 --- a/src/transformers/models/mask2former/modeling_mask2former.py +++ b/src/transformers/models/mask2former/modeling_mask2former.py @@ -24,17 +24,10 @@ import torch from torch import Tensor, nn from ...activations import ACT2FN -from ...file_utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_scipy_available, - replace_return_docstrings, - requires_backends, -) +from ...file_utils import ModelOutput, is_scipy_available, requires_backends from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel -from ...utils import is_accelerate_available, logging +from ...utils import auto_docstring, is_accelerate_available, logging from ...utils.backbone_utils import load_backbone from .configuration_mask2former import Mask2FormerConfig @@ -49,11 +42,6 @@ if is_accelerate_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Mask2FormerConfig" -_CHECKPOINT_FOR_DOC = "facebook/mask2former-swin-small-coco-instance" -_IMAGE_PROCESSOR_FOR_DOC = "Mask2FormerImageProcessor" - - @dataclass class Mask2FormerPixelDecoderOutput(ModelOutput): """ @@ -2097,39 +2085,7 @@ class Mask2FormerTransformerModule(nn.Module): return decoder_output -MASK2FORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Mask2FormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MASK2FORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`AutoImageProcessor.preprocess`] for details. - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of Detr's decoder attention layers. - return_dict (`bool`, *optional*): - Whether or not to return a [`~Mask2FormerModelOutput`] instead of a plain tuple. -""" - - +@auto_docstring class Mask2FormerPreTrainedModel(PreTrainedModel): config_class = Mask2FormerConfig base_model_prefix = "model" @@ -2205,10 +2161,7 @@ class Mask2FormerPreTrainedModel(PreTrainedModel): nn.init.constant_(module.reference_points.bias.data, 0.0) -@add_start_docstrings( - "The bare Mask2Former Model outputting raw hidden-states without any specific head on top.", - MASK2FORMER_START_DOCSTRING, -) +@auto_docstring class Mask2FormerModel(Mask2FormerPreTrainedModel): main_input_name = "pixel_values" @@ -2219,8 +2172,7 @@ class Mask2FormerModel(Mask2FormerPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(MASK2FORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Mask2FormerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -2229,35 +2181,6 @@ class Mask2FormerModel(Mask2FormerPreTrainedModel): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Mask2FormerModelOutput: - r""" - Returns: - `Mask2FormerModelOutput` - - Examples: - ```python - >>> import torch - >>> from PIL import Image - >>> import requests - >>> from transformers import AutoImageProcessor, Mask2FormerModel - - >>> # load image - >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - - >>> # load image preprocessor and Mask2FormerModel trained on COCO instance segmentation dataset - >>> image_processor = AutoImageProcessor.from_pretrained("facebook/mask2former-swin-small-coco-instance") - >>> model = Mask2FormerModel.from_pretrained("facebook/mask2former-swin-small-coco-instance") - >>> inputs = image_processor(image, return_tensors="pt") - - >>> # forward pass - >>> with torch.no_grad(): - ... outputs = model(**inputs) - - >>> # model outputs last hidden states of shape (batch_size, num_queries, hidden_size) - >>> print(outputs.transformer_decoder_last_hidden_state.shape) - torch.Size([1, 100, 256]) - ``` - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -2309,9 +2232,10 @@ class Mask2FormerModel(Mask2FormerPreTrainedModel): return output -@add_start_docstrings( - "The Mask2Former Model with heads on top for instance/semantic/panoptic segmentation.", - MASK2FORMER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mask2Former Model with heads on top for instance/semantic/panoptic segmentation. + """ ) class Mask2FormerForUniversalSegmentation(Mask2FormerPreTrainedModel): main_input_name = "pixel_values" @@ -2366,8 +2290,7 @@ class Mask2FormerForUniversalSegmentation(Mask2FormerPreTrainedModel): return auxiliary_logits - @add_start_docstrings_to_model_forward(MASK2FORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Mask2FormerForUniversalSegmentationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -2385,9 +2308,8 @@ class Mask2FormerForUniversalSegmentation(Mask2FormerPreTrainedModel): class_labels (`List[torch.LongTensor]`, *optional*): list of target class labels of shape `(num_labels, height, width)` to be fed to a model. They identify the labels of `mask_labels`, e.g. the label of `mask_labels[i][j]` if `class_labels[i][j]`. - - Returns: - `Mask2FormerUniversalSegmentationOutput` + output_auxiliary_logits (`bool`, *optional*): + Whether or not to output auxiliary logits. Examples: diff --git a/src/transformers/models/maskformer/modeling_maskformer.py b/src/transformers/models/maskformer/modeling_maskformer.py index 60016172d2..4745e153c2 100644 --- a/src/transformers/models/maskformer/modeling_maskformer.py +++ b/src/transformers/models/maskformer/modeling_maskformer.py @@ -29,12 +29,10 @@ from ...modeling_outputs import BaseModelOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_accelerate_available, is_scipy_available, logging, - replace_return_docstrings, requires_backends, ) from ...utils.backbone_utils import load_backbone @@ -53,10 +51,6 @@ if is_scipy_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MaskFormerConfig" -_CHECKPOINT_FOR_DOC = "facebook/maskformer-swin-base-ade" - - @dataclass # Copied from transformers.models.detr.modeling_detr.DetrDecoderOutput class DetrDecoderOutput(BaseModelOutputWithCrossAttentions): @@ -1447,39 +1441,7 @@ class MaskFormerTransformerModule(nn.Module): return decoder_output -MASKFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MaskFormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MASKFORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`MaskFormerImageProcessor.__call__`] for details. - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of Detr's decoder attention layers. - return_dict (`bool`, *optional*): - Whether or not to return a [`~MaskFormerModelOutput`] instead of a plain tuple. -""" - - +@auto_docstring class MaskFormerPreTrainedModel(PreTrainedModel): config_class = MaskFormerConfig base_model_prefix = "model" @@ -1525,10 +1487,7 @@ class MaskFormerPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -@add_start_docstrings( - "The bare MaskFormer Model outputting raw hidden-states without any specific head on top.", - MASKFORMER_START_DOCSTRING, -) +@auto_docstring class MaskFormerModel(MaskFormerPreTrainedModel): def __init__(self, config: MaskFormerConfig): super().__init__(config) @@ -1539,8 +1498,7 @@ class MaskFormerModel(MaskFormerPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(MASKFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskFormerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -1550,8 +1508,6 @@ class MaskFormerModel(MaskFormerPreTrainedModel): return_dict: Optional[bool] = None, ) -> MaskFormerModelOutput: r""" - Returns: - Examples: ```python @@ -1710,8 +1666,7 @@ class MaskFormerForInstanceSegmentation(MaskFormerPreTrainedModel): return class_queries_logits, masks_queries_logits, auxiliary_logits - @add_start_docstrings_to_model_forward(MASKFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskFormerForInstanceSegmentationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -1729,8 +1684,8 @@ class MaskFormerForInstanceSegmentation(MaskFormerPreTrainedModel): class_labels (`List[torch.LongTensor]`, *optional*): list of target class labels of shape `(num_labels, height, width)` to be fed to a model. They identify the labels of `mask_labels`, e.g. the label of `mask_labels[i][j]` if `class_labels[i][j]`. - - Returns: + output_auxiliary_logits (`bool`, *optional*): + Whether or not to output auxiliary logits. Examples: diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py index 850fde60d1..560b8eca9b 100755 --- a/src/transformers/models/mbart/modeling_mbart.py +++ b/src/transformers/models/mbart/modeling_mbart.py @@ -42,14 +42,7 @@ from ...modeling_outputs import ( Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_mbart import MBartConfig @@ -59,12 +52,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/mbart-large-cc25" -_CONFIG_FOR_DOC = "MBartConfig" - -# Base model docstring -_EXPECTED_OUTPUT_SHAPE = [1, 8, 1024] - def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int): """ @@ -736,6 +723,7 @@ class MBartClassificationHead(nn.Module): return hidden_states +@auto_docstring class MBartPreTrainedModel(PreTrainedModel): config_class = MBartConfig base_model_prefix = "model" @@ -766,159 +754,6 @@ class MBartPreTrainedModel(PreTrainedModel): return dummy_inputs -MBART_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MBartConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MBART_GENERATION_EXAMPLE = r""" - Translation example: - - ```python - >>> from transformers import AutoTokenizer, MBartForConditionalGeneration - - >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-en-ro") - - >>> example_english_phrase = "42 is the answer" - >>> inputs = tokenizer(example_english_phrase, return_tensors="pt") - - >>> # Translate - >>> generated_ids = model.generate(**inputs, num_beams=4, max_length=5) - >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - '42 este răspuns' - ``` - - Mask filling example: - - ```python - >>> from transformers import AutoTokenizer, MBartForConditionalGeneration - - >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") - - >>> # de_DE is the language symbol id for German - >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE" - - >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="pt")["input_ids"] - >>> logits = model(input_ids).logits - - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() - >>> probs = logits[0, masked_index].softmax(dim=0) - >>> values, predictions = probs.topk(5) - - >>> tokenizer.decode(predictions).split() - ['nett', 'sehr', 'ganz', 'nicht', 'so'] - ``` -""" - -MBART_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - MBart uses a specific language id token as the starting token for `decoder_input_ids` generation that - varies according to source and target language, *e.g.* 25004 for *en_XX*, and 25003 for *de_DE*. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class MBartEncoder(MBartPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1394,10 +1229,7 @@ class MBartDecoder(MBartPreTrainedModel): ) -@add_start_docstrings( - "The bare MBART Model outputting raw hidden-states without any specific head on top.", - MBART_START_DOCSTRING, -) +@auto_docstring class MBartModel(MBartPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1433,13 +1265,7 @@ class MBartModel(MBartPreTrainedModel): self._tie_or_clone_weights(self.encoder.embed_tokens, self.get_input_embeddings()) self._tie_or_clone_weights(self.decoder.embed_tokens, self.get_input_embeddings()) - @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1458,6 +1284,33 @@ class MBartModel(MBartPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Seq2SeqModelOutput, Tuple[torch.FloatTensor]]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + MBart uses a specific language id token as the starting token for `decoder_input_ids` generation that + varies according to source and target language, *e.g.* 25004 for *en_XX*, and 25003 for *de_DE*. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1519,9 +1372,10 @@ class MBartModel(MBartPreTrainedModel): ) -@add_start_docstrings( - "The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models.", - MBART_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models. + """ ) class MBartForConditionalGeneration(MBartPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1565,9 +1419,7 @@ class MBartForConditionalGeneration(MBartPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(MBART_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1588,13 +1440,74 @@ class MBartForConditionalGeneration(MBartPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + MBart uses a specific language id token as the starting token for `decoder_input_ids` generation that + varies according to source and target language, *e.g.* 25004 for *en_XX*, and 25003 for *de_DE*. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Translation: + ```python + >>> from transformers import AutoTokenizer, MBartForConditionalGeneration + + >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-en-ro") + + >>> example_english_phrase = "42 is the answer" + >>> inputs = tokenizer(example_english_phrase, return_tensors="pt") + + >>> # Translate + >>> generated_ids = model.generate(**inputs, num_beams=4, max_length=5) + >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + '42 este răspuns' + ``` + + Mask filling example: + + ```python + >>> from transformers import AutoTokenizer, MBartForConditionalGeneration + + >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") + + >>> # de_DE is the language symbol id for German + >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE" + + >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="pt")["input_ids"] + >>> logits = model(input_ids).logits + + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = logits[0, masked_index].softmax(dim=0) + >>> values, predictions = probs.topk(5) + + >>> tokenizer.decode(predictions).split() + ['nett', 'sehr', 'ganz', 'nicht', 'so'] + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1660,12 +1573,11 @@ class MBartForConditionalGeneration(MBartPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MBart model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MBART_START_DOCSTRING, + """ ) class MBartForSequenceClassification(MBartPreTrainedModel): _tied_weights_keys = ["model.encoder.embed_tokens.weight", "model.decoder.embed_tokens.weight"] @@ -1683,12 +1595,7 @@ class MBartForSequenceClassification(MBartPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.bart.modeling_bart.BartForSequenceClassification.forward def forward( self, @@ -1709,6 +1616,33 @@ class MBartForSequenceClassification(MBartPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). @@ -1789,13 +1723,7 @@ class MBartForSequenceClassification(MBartPreTrainedModel): ) -@add_start_docstrings( - """ - MBART Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MBART_START_DOCSTRING, -) +@auto_docstring class MBartForQuestionAnswering(MBartPreTrainedModel): _tied_weights_keys = ["model.encoder.embed_tokens.weight", "model.decoder.embed_tokens.weight"] @@ -1811,12 +1739,7 @@ class MBartForQuestionAnswering(MBartPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqQuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.bart.modeling_bart.BartForQuestionAnswering.forward def forward( self, @@ -1838,14 +1761,33 @@ class MBartForQuestionAnswering(MBartPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if start_positions is not None and end_positions is not None: @@ -1962,7 +1904,7 @@ class MBartForCausalLM(MBartPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1980,72 +1922,15 @@ class MBartForCausalLM(MBartPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/megatron_bert/modeling_megatron_bert.py b/src/transformers/models/megatron_bert/modeling_megatron_bert.py index 35ae4a6516..557cdb1fee 100755 --- a/src/transformers/models/megatron_bert/modeling_megatron_bert.py +++ b/src/transformers/models/megatron_bert/modeling_megatron_bert.py @@ -41,22 +41,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_megatron_bert import MegatronBertConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MegatronBertConfig" -_CHECKPOINT_FOR_DOC = "nvidia/megatron-bert-cased-345m" - def load_tf_weights_in_megatron_bert(model, config, tf_checkpoint_path): """Load tf checkpoints in a pytorch model.""" @@ -698,12 +688,8 @@ class MegatronBertPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class MegatronBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MegatronBertConfig load_tf_weights = load_tf_weights_in_megatron_bert base_model_prefix = "bert" @@ -759,76 +745,7 @@ class MegatronBertForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -MEGATRON_BERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MegatronBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MEGATRON_BERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MegatronBert Model transformer outputting raw hidden-states without any specific head on top.", - MEGATRON_BERT_START_DOCSTRING, -) +@auto_docstring class MegatronBertModel(MegatronBertPreTrainedModel): """ @@ -843,6 +760,10 @@ class MegatronBertModel(MegatronBertPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -868,12 +789,7 @@ class MegatronBertModel(MegatronBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -890,26 +806,6 @@ class MegatronBertModel(MegatronBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -999,17 +895,20 @@ class MegatronBertModel(MegatronBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MegatronBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. - """, - MEGATRON_BERT_START_DOCSTRING, + """ ) class MegatronBertForPreTraining(MegatronBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder"] def __init__(self, config, add_binary_head=True): + r""" + add_binary_head (`bool`, *optional*, defaults to `True`): + Whether or not to add a binary head. + """ super().__init__(config) self.bert = MegatronBertModel(config) @@ -1025,8 +924,7 @@ class MegatronBertForPreTraining(MegatronBertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MegatronBertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1052,10 +950,6 @@ class MegatronBertForPreTraining(MegatronBertPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. - - Returns: Example: @@ -1109,9 +1003,10 @@ class MegatronBertForPreTraining(MegatronBertPreTrainedModel): ) -@add_start_docstrings( - """MegatronBert Model with a `language modeling` head on top for CLM fine-tuning.""", - MEGATRON_BERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + MegatronBert Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class MegatronBertForCausalLM(MegatronBertPreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder"] @@ -1135,8 +1030,7 @@ class MegatronBertForCausalLM(MegatronBertPreTrainedModel, GenerationMixin): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1156,30 +1050,10 @@ class MegatronBertForCausalLM(MegatronBertPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1249,7 +1123,7 @@ class MegatronBertForCausalLM(MegatronBertPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings("""MegatronBert Model with a `language modeling` head on top.""", MEGATRON_BERT_START_DOCSTRING) +@auto_docstring class MegatronBertForMaskedLM(MegatronBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder"] @@ -1275,12 +1149,7 @@ class MegatronBertForMaskedLM(MegatronBertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1354,9 +1223,10 @@ class MegatronBertForMaskedLM(MegatronBertPreTrainedModel): return {"input_ids": input_ids, "attention_mask": attention_mask} -@add_start_docstrings( - """MegatronBert Model with a `next sentence prediction (classification)` head on top.""", - MEGATRON_BERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + MegatronBert Model with a `next sentence prediction (classification)` head on top. + """ ) class MegatronBertForNextSentencePrediction(MegatronBertPreTrainedModel): def __init__(self, config): @@ -1368,8 +1238,7 @@ class MegatronBertForNextSentencePrediction(MegatronBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=NextSentencePredictorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1392,8 +1261,6 @@ class MegatronBertForNextSentencePrediction(MegatronBertPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - Returns: - Example: ```python @@ -1455,12 +1322,11 @@ class MegatronBertForNextSentencePrediction(MegatronBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MegatronBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MEGATRON_BERT_START_DOCSTRING, + """ ) class MegatronBertForSequenceClassification(MegatronBertPreTrainedModel): def __init__(self, config): @@ -1474,12 +1340,7 @@ class MegatronBertForSequenceClassification(MegatronBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1552,13 +1413,7 @@ class MegatronBertForSequenceClassification(MegatronBertPreTrainedModel): ) -@add_start_docstrings( - """ - MegatronBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output - and a softmax) e.g. for RocStories/SWAG tasks. - """, - MEGATRON_BERT_START_DOCSTRING, -) +@auto_docstring class MegatronBertForMultipleChoice(MegatronBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1570,14 +1425,7 @@ class MegatronBertForMultipleChoice(MegatronBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1592,6 +1440,30 @@ class MegatronBertForMultipleChoice(MegatronBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1645,13 +1517,7 @@ class MegatronBertForMultipleChoice(MegatronBertPreTrainedModel): ) -@add_start_docstrings( - """ - MegatronBert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - MEGATRON_BERT_START_DOCSTRING, -) +@auto_docstring class MegatronBertForTokenClassification(MegatronBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1664,12 +1530,7 @@ class MegatronBertForTokenClassification(MegatronBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1723,13 +1584,7 @@ class MegatronBertForTokenClassification(MegatronBertPreTrainedModel): ) -@add_start_docstrings( - """ - MegatronBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MEGATRON_BERT_START_DOCSTRING, -) +@auto_docstring class MegatronBertForQuestionAnswering(MegatronBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1741,12 +1596,7 @@ class MegatronBertForQuestionAnswering(MegatronBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MEGATRON_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1761,16 +1611,6 @@ class MegatronBertForQuestionAnswering(MegatronBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.bert( diff --git a/src/transformers/models/mgp_str/modeling_mgp_str.py b/src/transformers/models/mgp_str/modeling_mgp_str.py index 684cfd2e78..5d19456c3c 100644 --- a/src/transformers/models/mgp_str/modeling_mgp_str.py +++ b/src/transformers/models/mgp_str/modeling_mgp_str.py @@ -25,25 +25,12 @@ from torch import nn from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_mgp_str import MgpstrConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "MgpstrConfig" -_TOKENIZER_FOR_DOC = "MgpstrTokenizer" - -# Base docstring -_CHECKPOINT_FOR_DOC = "alibaba-damo/mgp-str-base" - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: @@ -306,12 +293,8 @@ class MgpstrA3Module(nn.Module): return (a3_out, attentions) +@auto_docstring class MgpstrPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MgpstrConfig base_model_prefix = "mgp_str" _no_split_modules = [] @@ -330,37 +313,7 @@ class MgpstrPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MGP_STR_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MgpstrConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MGP_STR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MGP-STR Model transformer outputting raw hidden-states without any specific head on top.", - MGP_STR_START_DOCSTRING, -) +@auto_docstring class MgpstrModel(MgpstrPreTrainedModel): def __init__(self, config: MgpstrConfig): super().__init__(config) @@ -371,7 +324,7 @@ class MgpstrModel(MgpstrPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.embeddings.proj - @add_start_docstrings_to_model_forward(MGP_STR_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -406,12 +359,11 @@ class MgpstrModel(MgpstrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MGP-STR Model transformer with three classification heads on top (three A^3 modules and three linear layer on top of the transformer encoder output) for scene text recognition (STR) . - """, - MGP_STR_START_DOCSTRING, + """ ) class MgpstrForSceneTextRecognition(MgpstrPreTrainedModel): config_class = MgpstrConfig @@ -431,8 +383,7 @@ class MgpstrForSceneTextRecognition(MgpstrPreTrainedModel): self.bpe_head = nn.Linear(config.hidden_size, config.num_bpe_labels) self.wp_head = nn.Linear(config.hidden_size, config.num_wordpiece_labels) - @add_start_docstrings_to_model_forward(MGP_STR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MgpstrModelOutput, config_class=MgpstrConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -446,8 +397,6 @@ class MgpstrForSceneTextRecognition(MgpstrPreTrainedModel): Whether or not to return the attentions tensors of a3 modules. See `a3_attentions` under returned tensors for more detail. - Returns: - Example: ```python diff --git a/src/transformers/models/mimi/modeling_mimi.py b/src/transformers/models/mimi/modeling_mimi.py index bfe6698c55..96f879d31f 100644 --- a/src/transformers/models/mimi/modeling_mimi.py +++ b/src/transformers/models/mimi/modeling_mimi.py @@ -29,14 +29,7 @@ from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, is_torch_flex_attn_available, logging from .configuration_mimi import MimiConfig @@ -52,10 +45,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "MimiConfig" - - @dataclass class MimiOutput(ModelOutput): """ @@ -1437,12 +1426,8 @@ class MimiSplitResidualVectorQuantizer(nn.Module): return quantized_out +@auto_docstring class MimiPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MimiConfig base_model_prefix = "mimi" main_input_name = "input_values" @@ -1472,58 +1457,10 @@ class MimiPreTrainedModel(PreTrainedModel): module.scale.data.fill_(self.config.layer_scale_initial_scale) -MIMI_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MimiConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -MIMI_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): - Raw audio input converted to Float. - padding_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0 - for *masked*. - num_quantizers (`int`, *optional*): - Number of quantizers (i.e codebooks) to use. By default, all quantizers are used. - audio_codes (`torch.LongTensor` of shape `(batch_size, num_quantizers, codes_length)`, *optional*): - Discret code embeddings computed using `model.encode`. - encoder_past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the encoder transformer. - This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - The model will output the same cache format that is fed as input. - - If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't - have their past key value states given to this model). - decoder_past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the decoder transformer. - This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - The model will output the same cache format that is fed as input. - - If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't - have their past key value states given to this model). - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The Mimi neural audio codec model.", - MIMI_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mimi neural audio codec model. + """ ) class MimiModel(MimiPreTrainedModel): def __init__(self, config: MimiConfig): @@ -1766,8 +1703,7 @@ class MimiModel(MimiPreTrainedModel): ) return MimiDecoderOutput(audio_values, decoder_past_key_values) - @add_start_docstrings_to_model_forward(MIMI_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MimiOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: torch.Tensor, @@ -1779,7 +1715,31 @@ class MimiModel(MimiPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor, torch.Tensor], MimiOutput]: r""" - Returns: + input_values (`torch.FloatTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): + Raw audio input converted to Float. + padding_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* or 0 + for *masked*. + num_quantizers (`int`, *optional*): + Number of quantizers (i.e codebooks) to use. By default, all quantizers are used. + audio_codes (`torch.LongTensor` of shape `(batch_size, num_quantizers, codes_length)`, *optional*): + Discret code embeddings computed using `model.encode`. + encoder_past_key_values (`Cache`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the encoder transformer. + This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + The model will output the same cache format that is fed as input. + + If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't + have their past key value states given to this model). + decoder_past_key_values (`Cache`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding of the decoder transformer. + This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + The model will output the same cache format that is fed as input. + + If `past_key_values` are used, the user can optionally input only the last `audio_values` or `audio_codes (those that don't + have their past key value states given to this model). Examples: diff --git a/src/transformers/models/mistral/modeling_mistral.py b/src/transformers/models/mistral/modeling_mistral.py index 3575985232..ac7ed954b9 100644 --- a/src/transformers/models/mistral/modeling_mistral.py +++ b/src/transformers/models/mistral/modeling_mistral.py @@ -26,16 +26,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_mistral import MistralConfig @@ -47,9 +38,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "mistralai/Mistral-7B-v0.1" -_CONFIG_FOR_DOC = "MistralConfig" - class MistralMLP(nn.Module): def __init__(self, config): @@ -279,27 +267,7 @@ class MistralDecoderLayer(GradientCheckpointingLayer): return outputs -MISTRAL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MistralConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Mistral Model outputting raw hidden-states without any specific head on top.", - MISTRAL_START_DOCSTRING, -) +@auto_docstring class MistralPreTrainedModel(PreTrainedModel): config_class = MistralConfig base_model_prefix = "model" @@ -362,88 +330,8 @@ class MistralRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -MISTRAL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Mistral Model outputting raw hidden-states without any specific head on top.", - MISTRAL_START_DOCSTRING, -) +@auto_docstring class MistralModel(MistralPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MistralDecoderLayer`] - - Args: - config: MistralConfig - """ - def __init__(self, config: MistralConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -467,7 +355,7 @@ class MistralModel(MistralPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -720,6 +608,7 @@ class MistralModel(MistralPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class MistralForCausalLM(MistralPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -753,8 +642,7 @@ class MistralForCausalLM(MistralPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -771,19 +659,10 @@ class MistralForCausalLM(MistralPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -838,13 +717,7 @@ class MistralForCausalLM(MistralPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ - The Mistral Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - MISTRAL_START_DOCSTRING, -) +@auto_docstring class MistralForTokenClassification(MistralPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -869,12 +742,7 @@ class MistralForTokenClassification(MistralPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -920,8 +788,8 @@ class MistralForTokenClassification(MistralPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Mistral Model transformer with a sequence classification head on top (linear layer). [`MistralForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -932,8 +800,7 @@ class MistralForTokenClassification(MistralPreTrainedModel): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - MISTRAL_START_DOCSTRING, + """ ) class MistralForSequenceClassification(MistralPreTrainedModel): def __init__(self, config): @@ -952,7 +819,7 @@ class MistralForSequenceClassification(MistralPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1021,13 +888,7 @@ class MistralForSequenceClassification(MistralPreTrainedModel): ) -@add_start_docstrings( - """ -The Mistral Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MISTRAL_START_DOCSTRING, -) +@auto_docstring class MistralForQuestionAnswering(MistralPreTrainedModel): base_model_prefix = "model" @@ -1046,7 +907,7 @@ class MistralForQuestionAnswering(MistralPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/mistral3/modeling_mistral3.py b/src/transformers/models/mistral3/modeling_mistral3.py index 24cc21af60..3c57c816fa 100644 --- a/src/transformers/models/mistral3/modeling_mistral3.py +++ b/src/transformers/models/mistral3/modeling_mistral3.py @@ -34,19 +34,14 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import ( LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, is_torchdynamo_compiling, - replace_return_docstrings, ) from ..auto import AutoModel from .configuration_mistral3 import Mistral3Config -_CONFIG_FOR_DOC = "Mistra3Config" - - @use_kernel_forward_from_hub("RMSNorm") class Mistral3RMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -205,27 +200,7 @@ class Mistral3ModelOutputWithPast(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None -MISTRAL3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Mistral3Config`] or [`Mistral3VisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Mistral3 Model outputting raw hidden-states without any specific head on top.", - MISTRAL3_START_DOCSTRING, -) +@auto_docstring class Mistral3PreTrainedModel(PreTrainedModel): config_class = Mistral3Config base_model_prefix = "" @@ -255,9 +230,10 @@ class Mistral3PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -@add_start_docstrings( - """The Mistral3 model which consists of a vision backbone and a language model, without a language modeling head.""", - MISTRAL3_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mistral3 model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class Mistral3Model(Mistral3PreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -313,6 +289,7 @@ class Mistral3Model(Mistral3PreTrainedModel): return image_features @can_return_tuple + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -393,86 +370,10 @@ class Mistral3Model(Mistral3PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -MISTRAL3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`Mistral3Processor`] uses - [`CLIPImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The MISTRAL3 model which consists of a vision backbone and a language model.""", - MISTRAL3_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The MISTRAL3 model which consists of a vision backbone and a language model. + """ ) class Mistral3ForConditionalGeneration(Mistral3PreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -515,8 +416,7 @@ class Mistral3ForConditionalGeneration(Mistral3PreTrainedModel, GenerationMixin) return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Mistral3CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -536,20 +436,10 @@ class Mistral3ForConditionalGeneration(Mistral3PreTrainedModel, GenerationMixin) **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Mistral3CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/mistral3/modular_mistral3.py b/src/transformers/models/mistral3/modular_mistral3.py index 11dcfa2c3a..7494afdddc 100644 --- a/src/transformers/models/mistral3/modular_mistral3.py +++ b/src/transformers/models/mistral3/modular_mistral3.py @@ -21,13 +21,7 @@ from torch import nn from ...activations import ACT2FN from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import is_torchdynamo_compiling, logging from ..llava.modeling_llava import ( KwargsForCausalLM, LlavaCausalLMOutputWithPast, @@ -42,9 +36,6 @@ from .configuration_mistral3 import Mistral3Config logger = logging.get_logger(__name__) -MISTRAL3_INPUTS_DOCSTRING = None -_CONFIG_FOR_DOC = "Mistra3Config" - class Mistral3RMSNorm(MistralRMSNorm): pass @@ -177,7 +168,6 @@ class Mistral3Model(LlavaModel): image_features = self.multi_modal_projector(selected_image_feature.squeeze(0), image_sizes) return image_features - @can_return_tuple def forward( self, input_ids: torch.LongTensor = None, @@ -256,9 +246,6 @@ class Mistral3Model(LlavaModel): class Mistral3ForConditionalGeneration(LlavaForConditionalGeneration): - @can_return_tuple - @add_start_docstrings_to_model_forward(MISTRAL3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Mistral3CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: torch.LongTensor = None, @@ -278,20 +265,10 @@ class Mistral3ForConditionalGeneration(LlavaForConditionalGeneration): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, Mistral3CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/mixtral/modeling_mixtral.py b/src/transformers/models/mixtral/modeling_mixtral.py index e8012a8d9e..23a7c663ed 100644 --- a/src/transformers/models/mixtral/modeling_mixtral.py +++ b/src/transformers/models/mixtral/modeling_mixtral.py @@ -39,7 +39,6 @@ from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import ( BaseModelOutputWithPast, - CausalLMOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast, QuestionAnsweringModelOutput, @@ -49,16 +48,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_mixtral import MixtralConfig @@ -70,9 +60,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "mistralai/Mixtral-8x7B-v0.1" -_CONFIG_FOR_DOC = "MixtralConfig" - class MixtralBlockSparseTop2MLP(nn.Module): def __init__(self, config: MixtralConfig): @@ -435,27 +422,7 @@ class MixtralRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -MIXTRAL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MixtralConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Mixtral Model outputting raw hidden-states without any specific head on top.", - MIXTRAL_START_DOCSTRING, -) +@auto_docstring class MixtralPreTrainedModel(PreTrainedModel): config_class = MixtralConfig base_model_prefix = "model" @@ -484,88 +451,8 @@ class MixtralPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MIXTRAL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Mixtral Model outputting raw hidden-states without any specific head on top.", - MIXTRAL_START_DOCSTRING, -) +@auto_docstring class MixtralModel(MixtralPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MixtralDecoderLayer`] - - Args: - config: MixtralConfig - """ - def __init__(self, config: MixtralConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -589,7 +476,7 @@ class MixtralModel(MixtralPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -944,6 +831,7 @@ def load_balancing_loss_func( return overall_loss * num_experts +@auto_docstring class MixtralForCausalLM(MixtralPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -980,8 +868,7 @@ class MixtralForCausalLM(MixtralPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -999,19 +886,10 @@ class MixtralForCausalLM(MixtralPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1085,8 +963,8 @@ class MixtralForCausalLM(MixtralPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Mixtral Model transformer with a sequence classification head on top (linear layer). [`MixtralForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1097,8 +975,7 @@ class MixtralForCausalLM(MixtralPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - MIXTRAL_START_DOCSTRING, + """ ) class MixtralForSequenceClassification(MixtralPreTrainedModel): def __init__(self, config): @@ -1117,7 +994,7 @@ class MixtralForSequenceClassification(MixtralPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1186,13 +1063,7 @@ class MixtralForSequenceClassification(MixtralPreTrainedModel): ) -@add_start_docstrings( - """ - The Mixtral Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - MIXTRAL_START_DOCSTRING, -) +@auto_docstring class MixtralForTokenClassification(MixtralPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1217,12 +1088,7 @@ class MixtralForTokenClassification(MixtralPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1268,13 +1134,7 @@ class MixtralForTokenClassification(MixtralPreTrainedModel): ) -@add_start_docstrings( - """ -The Mixtral Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MIXTRAL_START_DOCSTRING, -) +@auto_docstring class MixtralForQuestionAnswering(MixtralPreTrainedModel): base_model_prefix = "model" @@ -1293,7 +1153,7 @@ class MixtralForQuestionAnswering(MixtralPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MIXTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/mixtral/modular_mixtral.py b/src/transformers/models/mixtral/modular_mixtral.py index f85836b567..726d2e9958 100644 --- a/src/transformers/models/mixtral/modular_mixtral.py +++ b/src/transformers/models/mixtral/modular_mixtral.py @@ -30,15 +30,9 @@ from torch import nn from ...activations import ACT2FN from ...cache_utils import DynamicCache from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - MoeCausalLMOutputWithPast, - MoeModelOutputWithPast, -) +from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - logging, -) +from ...utils import LossKwargs, logging from ..mistral.modeling_mistral import ( MistralAttention, MistralForCausalLM, @@ -55,9 +49,6 @@ from .configuration_mixtral import MixtralConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "mistralai/Mixtral-8x7B-v0.1" -_CONFIG_FOR_DOC = "MixtralConfig" - def load_balancing_loss_func( gate_logits: Union[torch.Tensor, Tuple[torch.Tensor], None], @@ -475,19 +466,10 @@ class MixtralForCausalLM(MistralForCausalLM): **kwargs: Unpack[KwargsForCausalLM], ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/mlcd/modeling_mlcd.py b/src/transformers/models/mlcd/modeling_mlcd.py index 851cd01a1a..d4f1441642 100644 --- a/src/transformers/models/mlcd/modeling_mlcd.py +++ b/src/transformers/models/mlcd/modeling_mlcd.py @@ -28,14 +28,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, can_return_tuple, logging, torch_int from .configuration_mlcd import MLCDVisionConfig @@ -465,22 +458,6 @@ class MLCDEncoder(nn.Module): ) -MLCD_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class MLCDVisionTransformer(nn.Module): def __init__(self, config: MLCDVisionConfig): super().__init__() @@ -494,7 +471,7 @@ class MLCDVisionTransformer(nn.Module): self.vision_rotary_embedding = MLCDRotaryEmbedding(config.hidden_size // config.num_attention_heads // 2) self.class_pos_emb = nn.Parameter(torch.randn(1, config.hidden_size // config.num_attention_heads // 2)) - @add_start_docstrings_to_model_forward(MLCD_VISION_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -502,10 +479,6 @@ class MLCDVisionTransformer(nn.Module): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -549,12 +522,8 @@ class MLCDVisionTransformer(nn.Module): ) +@auto_docstring class MLCDPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MLCDVisionConfig base_model_prefix = "mlcd" supports_gradient_checkpointing = True @@ -593,26 +562,10 @@ class MLCDPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -MLCD_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MLCDVisionConfig`]): - Model configuration class with all the parameters of the vision encoder. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare MLCD vision encoder outputting raw hidden-states without any specific head on top.", - MLCD_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from M_L_C_D without any head or projection on top. + """ ) class MLCDVisionModel(MLCDPreTrainedModel): config_class = MLCDVisionConfig @@ -628,8 +581,7 @@ class MLCDVisionModel(MLCDPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(MLCD_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=MLCDVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -638,9 +590,7 @@ class MLCDVisionModel(MLCDPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - - Examples: + Example: ```python >>> import requests diff --git a/src/transformers/models/mlcd/modular_mlcd.py b/src/transformers/models/mlcd/modular_mlcd.py index 60ff12d827..7e55288b41 100644 --- a/src/transformers/models/mlcd/modular_mlcd.py +++ b/src/transformers/models/mlcd/modular_mlcd.py @@ -19,21 +19,10 @@ import torch.nn as nn from ...configuration_utils import PretrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - BaseModelOutput, - BaseModelOutputWithPooling, -) -from ...modeling_utils import ( - ALL_ATTENTION_FUNCTIONS, - PreTrainedModel, -) +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..clip.modeling_clip import ( CLIPMLP, CLIPAttention, @@ -44,10 +33,7 @@ from ..clip.modeling_clip import ( CLIPVisionTransformer, ) from ..llama.modeling_llama import eager_attention_forward -from ..qwen2_vl.modeling_qwen2_vl import ( - VisionRotaryEmbedding, - apply_rotary_pos_emb_vision, -) +from ..qwen2_vl.modeling_qwen2_vl import VisionRotaryEmbedding, apply_rotary_pos_emb_vision logger = logging.get_logger(__name__) @@ -409,29 +395,13 @@ class MLCDEncoder(CLIPEncoder): ) -MLCD_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class MLCDVisionTransformer(CLIPVisionTransformer): def __init__(self, config: MLCDVisionConfig): super().__init__(config) self.vision_rotary_embedding = MLCDRotaryEmbedding(config.hidden_size // config.num_attention_heads // 2) self.class_pos_emb = nn.Parameter(torch.randn(1, config.hidden_size // config.num_attention_heads // 2)) - @add_start_docstrings_to_model_forward(MLCD_VISION_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -482,29 +452,8 @@ class MLCDVisionTransformer(CLIPVisionTransformer): ) -MLCD_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MLCDVisionConfig`]): - Model configuration class with all the parameters of the vision encoder. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - +@auto_docstring class MLCDPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MLCDVisionConfig base_model_prefix = "mlcd" supports_gradient_checkpointing = True @@ -543,13 +492,8 @@ class MLCDPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -@add_start_docstrings( - "The bare MLCD vision encoder outputting raw hidden-states without any specific head on top.", - MLCD_START_DOCSTRING, -) class MLCDVisionModel(CLIPVisionModel): - @add_start_docstrings_to_model_forward(MLCD_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=MLCDVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -558,6 +502,8 @@ class MLCDVisionModel(CLIPVisionModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" + Example: + ```python >>> import requests >>> from PIL import Image diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index 300b172c2d..b9d266ae47 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -31,15 +31,7 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, Causal from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_mllama import MllamaConfig, MllamaTextConfig, MllamaVisionConfig @@ -855,6 +847,7 @@ class MllamaRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@auto_docstring class MllamaPreTrainedModel(PreTrainedModel): config_class = MllamaConfig base_model_prefix = "model" @@ -1028,251 +1021,10 @@ class MllamaPreTrainedModel(PreTrainedModel): return causal_mask -MLLAMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MllamaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -MLLAMA_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, max_num_images, max_num_tiles, channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`MllamaImageProcessor.__call__`] for details ([]`MllamaProcessor`] uses - [`MllamaImageProcessor`] for processing images). - aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): - Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: - - - 1 for tiles that are **not masked**, - - 0 for tiles that are **masked**. - aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): - Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. - These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. - - For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: - - An image with aspect ratio [1, 1] would have ID 1 - - An image with aspect ratio [1, 2] would have ID 2 - - An image with aspect ratio [2, 1] would have ID 3 - - The id 0 is reserved for padding (i.e., no image). - - If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -MLLAMA_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): - Cross-attention mask to control the interaction between text tokens and image tiles. - This 4D tensor defines which image tiles each text token should attend to. - - For each text token (in seq_length): - - 1 indicates the token **should attend** to the corresponding image tile - - 0 indicates the token **should not attend** to the corresponding image tile - cross_attention_states (`torch.FloatTensor`, *optional*): - Output of the vision model, used for cross-attention. This tensor contains the processed image features that - the language model will attend to. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -MLLAMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, max_num_images, max_num_tiles, channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`MllamaImageProcessor.__call__`] for details ([]`MllamaProcessor`] uses - [`MllamaImageProcessor`] for processing images). - aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): - Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: - - - 1 for tiles that are **not masked**, - - 0 for tiles that are **masked**. - aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): - Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. - These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. - - For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: - - An image with aspect ratio [1, 1] would have ID 1 - - An image with aspect ratio [1, 2] would have ID 2 - - An image with aspect ratio [2, 1] would have ID 3 - - The id 0 is reserved for padding (i.e., no image). - - If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): - Cross-attention mask to control the interaction between text tokens and image tiles. - This 4D tensor defines which image tiles each text token should attend to. - - For each text token (in seq_length): - - 1 indicates the token **should attend** to the corresponding image tile - - 0 indicates the token **should not attend** to the corresponding image tile - cross_attention_states (`torch.FloatTensor`, *optional*): - Output of the vision model, used for cross-attention. This tensor contains the processed image features that - the language model will attend to. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The Mllama Vision Model which consists of two vision encoders.""", - MLLAMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mllama Vision Model which consists of two vision encoders. + """ ) class MllamaVisionModel(MllamaPreTrainedModel): config_class = MllamaVisionConfig @@ -1327,8 +1079,7 @@ class MllamaVisionModel(MllamaPreTrainedModel): hidden_state = torch.cat([class_embedding, hidden_state], dim=1) return hidden_state - @add_start_docstrings_to_model_forward(MLLAMA_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class="MllamaVisionConfig") + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -1339,8 +1090,23 @@ class MllamaVisionModel(MllamaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[BaseModelOutput, Tuple[torch.Tensor, ...]]: r""" + aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): + Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. + These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. - Returns: + For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: + - An image with aspect ratio [1, 1] would have ID 1 + - An image with aspect ratio [1, 2] would have ID 2 + - An image with aspect ratio [2, 1] would have ID 3 + + The id 0 is reserved for padding (i.e., no image). + + If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. + aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): + Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: + + - 1 for tiles that are **not masked**, + - 0 for tiles that are **masked**. Example: @@ -1486,9 +1252,10 @@ class MllamaVisionModel(MllamaPreTrainedModel): ) -@add_start_docstrings( - """The Mllama Text Model which consists of transformer with self and cross attention layers.""", - MLLAMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mllama Text Model which consists of transformer with self and cross attention layers. + """ ) class MllamaTextModel(MllamaPreTrainedModel): config_class = MllamaTextConfig @@ -1520,8 +1287,7 @@ class MllamaTextModel(MllamaPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(MLLAMA_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPast, config_class="MllamaTextConfig") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1539,9 +1305,26 @@ class MllamaTextModel(MllamaPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, BaseModelOutputWithPast]: - """ + r""" + cross_attention_states (`torch.FloatTensor`, *optional*): + Output of the vision model, used for cross-attention. This tensor contains the processed image features that + the language model will attend to. + cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): + Cross-attention mask to control the interaction between text tokens and image tiles. + This 4D tensor defines which image tiles each text token should attend to. - Returns: + For each text token (in seq_length): + - 1 indicates the token **should attend** to the corresponding image tile + - 0 indicates the token **should not attend** to the corresponding image tile + full_text_row_masked_out_mask (`Tuple[torch.Tensor, torch.Tensor]`, *optional*): + A tuple containing two tensors that mask out rows in the cross-attention mechanism: + - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1. + A value of 0 indicates that the corresponding text token's entire row in the cross-attention + matrix should be masked out (all image tokens ignored). + - The second tensor has the same shape and is used internally to apply the masking during + the forward pass of cross-attention layers. + This mask is derived from the cross_attention_mask and is used to handle cases where a text token + should not attend to any image token. Example: @@ -1680,9 +1463,10 @@ class MllamaTextModel(MllamaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The Mllama Text Model with a language modeling head on top.""", - MLLAMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mllama Text Model with a language modeling head on top. + """ ) class MllamaForCausalLM(MllamaPreTrainedModel, GenerationMixin): config_class = MllamaTextConfig @@ -1717,8 +1501,7 @@ class MllamaForCausalLM(MllamaPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(MLLAMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class="MllamaTextConfig") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1739,19 +1522,29 @@ class MllamaForCausalLM(MllamaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + cross_attention_states (`torch.FloatTensor`, *optional*): + Output of the vision model, used for cross-attention. This tensor contains the processed image features that + the language model will attend to. + cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): + Cross-attention mask to control the interaction between text tokens and image tiles. + This 4D tensor defines which image tiles each text token should attend to. - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + For each text token (in seq_length): + - 1 indicates the token **should attend** to the corresponding image tile + - 0 indicates the token **should not attend** to the corresponding image tile + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + full_text_row_masked_out_mask (`Tuple[torch.Tensor, torch.Tensor]`, *optional*): + A tuple containing two tensors that mask out rows in the cross-attention mechanism: + - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1. + A value of 0 indicates that the corresponding text token's entire row in the cross-attention + matrix should be masked out (all image tokens ignored). + - The second tensor has the same shape and is used internally to apply the masking during + the forward pass of cross-attention layers. + This mask is derived from the cross_attention_mask and is used to handle cases where a text token + should not attend to any image token. Example: @@ -1817,9 +1610,10 @@ class MllamaForCausalLM(MllamaPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """The Mllama model which consists of a vision encoder and a language model without language modeling head.""", - MLLAMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mllama model which consists of a vision encoder and a language model without language modeling head. + """ ) class MllamaModel(MllamaPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -1849,7 +1643,7 @@ class MllamaModel(MllamaPreTrainedModel): self.language_model.set_input_embeddings(value) @can_return_tuple - @add_start_docstrings_to_model_forward(MLLAMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1869,6 +1663,35 @@ class MllamaModel(MllamaPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): + Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: + + - 1 for tiles that are **not masked**, + - 0 for tiles that are **masked**. + aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): + Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. + These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. + + For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: + - An image with aspect ratio [1, 1] would have ID 1 + - An image with aspect ratio [1, 2] would have ID 2 + - An image with aspect ratio [2, 1] would have ID 3 + + The id 0 is reserved for padding (i.e., no image). + + If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. + cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): + Cross-attention mask to control the interaction between text tokens and image tiles. + This 4D tensor defines which image tiles each text token should attend to. + + For each text token (in seq_length): + - 1 indicates the token **should attend** to the corresponding image tile + - 0 indicates the token **should not attend** to the corresponding image tile + cross_attention_states (`torch.FloatTensor`, *optional*): + Output of the vision model, used for cross-attention. This tensor contains the processed image features that + the language model will attend to. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1941,9 +1764,10 @@ class MllamaModel(MllamaPreTrainedModel): ) -@add_start_docstrings( - """The Mllama model which consists of a vision encoder and a language model.""", - MLLAMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Mllama model which consists of a vision encoder and a language model. + """, ) class MllamaForConditionalGeneration(MllamaPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -1983,8 +1807,7 @@ class MllamaForConditionalGeneration(MllamaPreTrainedModel, GenerationMixin): return self.model.vision_model @can_return_tuple - @add_start_docstrings_to_model_forward(MLLAMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class="MllamaConfig") + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2007,20 +1830,37 @@ class MllamaForConditionalGeneration(MllamaPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): + Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). + - 1 for tiles that are **not masked**, + - 0 for tiles that are **masked**. + aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): + Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. + These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. + For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: + - An image with aspect ratio [1, 1] would have ID 1 + - An image with aspect ratio [1, 2] would have ID 2 + - An image with aspect ratio [2, 1] would have ID 3 - Returns: + The id 0 is reserved for padding (i.e., no image). + + If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. + cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): + Cross-attention mask to control the interaction between text tokens and image tiles. + This 4D tensor defines which image tiles each text token should attend to. + + For each text token (in seq_length): + - 1 indicates the token **should attend** to the corresponding image tile + - 0 indicates the token **should not attend** to the corresponding image tile + cross_attention_states (`torch.FloatTensor`, *optional*): + Output of the vision model, used for cross-attention. This tensor contains the processed image features that + the language model will attend to. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/mobilebert/modeling_mobilebert.py b/src/transformers/models/mobilebert/modeling_mobilebert.py index b1d269055e..7c0c18a8df 100644 --- a/src/transformers/models/mobilebert/modeling_mobilebert.py +++ b/src/transformers/models/mobilebert/modeling_mobilebert.py @@ -43,39 +43,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_mobilebert import MobileBertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/mobilebert-uncased" -_CONFIG_FOR_DOC = "MobileBertConfig" - -# TokenClassification docstring -_CHECKPOINT_FOR_TOKEN_CLASSIFICATION = "mrm8488/mobilebert-finetuned-ner" -_TOKEN_CLASS_EXPECTED_OUTPUT = "['I-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'I-LOC', 'I-LOC']" -_TOKEN_CLASS_EXPECTED_LOSS = 0.03 - -# QuestionAnswering docstring -_CHECKPOINT_FOR_QA = "csarron/mobilebert-uncased-squad-v2" -_QA_EXPECTED_OUTPUT = "'a nice puppet'" -_QA_EXPECTED_LOSS = 3.98 -_QA_TARGET_START_INDEX = 12 -_QA_TARGET_END_INDEX = 13 - -# SequenceClassification docstring -_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "lordtt13/emo-mobilebert" -_SEQ_CLASS_EXPECTED_OUTPUT = "'others'" -_SEQ_CLASS_EXPECTED_LOSS = "4.72" - def load_tf_weights_in_mobilebert(model, config, tf_checkpoint_path): """Load tf checkpoints in a pytorch model.""" @@ -679,12 +652,8 @@ class MobileBertPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class MobileBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MobileBertConfig load_tf_weights = load_tf_weights_in_mobilebert base_model_prefix = "mobilebert" @@ -742,82 +711,17 @@ class MobileBertForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -MOBILEBERT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MobileBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MOBILEBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MobileBert Model transformer outputting raw hidden-states without any specific head on top.", - MOBILEBERT_START_DOCSTRING, -) +@auto_docstring class MobileBertModel(MobileBertPreTrainedModel): """ https://arxiv.org/pdf/2004.02984.pdf """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config self.embeddings = MobileBertEmbeddings(config) @@ -842,12 +746,7 @@ class MobileBertModel(MobileBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -919,12 +818,11 @@ class MobileBertModel(MobileBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. - """, - MOBILEBERT_START_DOCSTRING, + """ ) class MobileBertForPreTraining(MobileBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -952,8 +850,7 @@ class MobileBertForPreTraining(MobileBertPreTrainedModel): return super().resize_token_embeddings(new_num_tokens=new_num_tokens) - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MobileBertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -980,8 +877,6 @@ class MobileBertForPreTraining(MobileBertPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - Returns: - Examples: ```python @@ -1034,7 +929,7 @@ class MobileBertForPreTraining(MobileBertPreTrainedModel): ) -@add_start_docstrings("""MobileBert Model with a `language modeling` head on top.""", MOBILEBERT_START_DOCSTRING) +@auto_docstring class MobileBertForMaskedLM(MobileBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -1061,14 +956,7 @@ class MobileBertForMaskedLM(MobileBertPreTrainedModel): ) return super().resize_token_embeddings(new_num_tokens=new_num_tokens) - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'paris'", - expected_loss=0.57, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1132,9 +1020,10 @@ class MobileBertOnlyNSPHead(nn.Module): return seq_relationship_score -@add_start_docstrings( - """MobileBert Model with a `next sentence prediction (classification)` head on top.""", - MOBILEBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + MobileBert Model with a `next sentence prediction (classification)` head on top. + """ ) class MobileBertForNextSentencePrediction(MobileBertPreTrainedModel): def __init__(self, config): @@ -1146,8 +1035,7 @@ class MobileBertForNextSentencePrediction(MobileBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=NextSentencePredictorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1170,8 +1058,6 @@ class MobileBertForNextSentencePrediction(MobileBertPreTrainedModel): - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. - Returns: - Examples: ```python @@ -1232,12 +1118,11 @@ class MobileBertForNextSentencePrediction(MobileBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MOBILEBERT_START_DOCSTRING, + """ ) # Copied from transformers.models.bert.modeling_bert.BertForSequenceClassification with Bert->MobileBert all-casing class MobileBertForSequenceClassification(MobileBertPreTrainedModel): @@ -1256,14 +1141,7 @@ class MobileBertForSequenceClassification(MobileBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1336,13 +1214,7 @@ class MobileBertForSequenceClassification(MobileBertPreTrainedModel): ) -@add_start_docstrings( - """ - MobileBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MOBILEBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.bert.modeling_bert.BertForQuestionAnswering with Bert->MobileBert all-casing class MobileBertForQuestionAnswering(MobileBertPreTrainedModel): def __init__(self, config): @@ -1355,16 +1227,7 @@ class MobileBertForQuestionAnswering(MobileBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_QA, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - qa_target_start_index=_QA_TARGET_START_INDEX, - qa_target_end_index=_QA_TARGET_END_INDEX, - expected_output=_QA_EXPECTED_OUTPUT, - expected_loss=_QA_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1379,16 +1242,6 @@ class MobileBertForQuestionAnswering(MobileBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.mobilebert( @@ -1440,13 +1293,7 @@ class MobileBertForQuestionAnswering(MobileBertPreTrainedModel): ) -@add_start_docstrings( - """ - MobileBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and - a softmax) e.g. for RocStories/SWAG tasks. - """, - MOBILEBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.bert.modeling_bert.BertForMultipleChoice with Bert->MobileBert all-casing class MobileBertForMultipleChoice(MobileBertPreTrainedModel): def __init__(self, config): @@ -1462,14 +1309,7 @@ class MobileBertForMultipleChoice(MobileBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1484,6 +1324,30 @@ class MobileBertForMultipleChoice(MobileBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1537,13 +1401,7 @@ class MobileBertForMultipleChoice(MobileBertPreTrainedModel): ) -@add_start_docstrings( - """ - MobileBert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - MOBILEBERT_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.bert.modeling_bert.BertForTokenClassification with Bert->MobileBert all-casing class MobileBertForTokenClassification(MobileBertPreTrainedModel): def __init__(self, config): @@ -1560,14 +1418,7 @@ class MobileBertForTokenClassification(MobileBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_TOKEN_CLASSIFICATION, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_TOKEN_CLASS_EXPECTED_OUTPUT, - expected_loss=_TOKEN_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1_fast.py b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1_fast.py index 2da7a1be79..e716553a6d 100644 --- a/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1_fast.py +++ b/src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1_fast.py @@ -15,19 +15,15 @@ """Fast Image processor class for MobileNetV1.""" from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, Unpack, ) from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast MobileNetV1 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class MobileNetV1ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN diff --git a/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py b/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py index a315ccd326..b77dd75ebf 100755 --- a/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py +++ b/src/transformers/models/mobilenet_v1/modeling_mobilenet_v1.py @@ -23,25 +23,13 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_mobilenet_v1 import MobileNetV1Config logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "MobileNetV1Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "google/mobilenet_v1_1.0_224" -_EXPECTED_OUTPUT_SHAPE = [1, 1024, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "google/mobilenet_v1_1.0_224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - - def _build_tf_to_pytorch_map(model, config, tf_weights=None): """ A map of modules from TF to PyTorch. @@ -239,12 +227,8 @@ class MobileNetV1ConvLayer(nn.Module): return features +@auto_docstring class MobileNetV1PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MobileNetV1Config load_tf_weights = load_tf_weights_in_mobilenet_v1 base_model_prefix = "mobilenet_v1" @@ -263,36 +247,13 @@ class MobileNetV1PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MOBILENET_V1_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MOBILENET_V1_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`MobileNetV1ImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MobileNetV1 model outputting raw hidden-states without any specific head on top.", - MOBILENET_V1_START_DOCSTRING, -) +@auto_docstring class MobileNetV1Model(MobileNetV1PreTrainedModel): def __init__(self, config: MobileNetV1Config, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -345,14 +306,7 @@ class MobileNetV1Model(MobileNetV1PreTrainedModel): def _prune_heads(self, heads_to_prune): raise NotImplementedError - @add_start_docstrings_to_model_forward(MOBILENET_V1_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -394,12 +348,11 @@ class MobileNetV1Model(MobileNetV1PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - MOBILENET_V1_START_DOCSTRING, + """ ) class MobileNetV1ForImageClassification(MobileNetV1PreTrainedModel): def __init__(self, config: MobileNetV1Config) -> None: @@ -417,13 +370,7 @@ class MobileNetV1ForImageClassification(MobileNetV1PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILENET_V1_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2_fast.py b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2_fast.py index aaa77f68a4..3edcd2d7b1 100644 --- a/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2_fast.py +++ b/src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2_fast.py @@ -16,19 +16,16 @@ from typing import List, Optional, Tuple -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling -from ...utils import add_start_docstrings, is_torch_available, is_torch_tensor +from ...utils import auto_docstring, is_torch_available, is_torch_tensor if is_torch_available(): import torch -@add_start_docstrings( - "Constructs a fast MobileNetV2 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class MobileNetV2ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN diff --git a/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py b/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py index bd36d111f8..f3cfc9098d 100755 --- a/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py +++ b/src/transformers/models/mobilenet_v2/modeling_mobilenet_v2.py @@ -27,31 +27,13 @@ from ...modeling_outputs import ( SemanticSegmenterOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_mobilenet_v2 import MobileNetV2Config logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "MobileNetV2Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "google/mobilenet_v2_1.0_224" -_EXPECTED_OUTPUT_SHAPE = [1, 1280, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "google/mobilenet_v2_1.0_224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - - def _build_tf_to_pytorch_map(model, config, tf_weights=None): """ A map of modules from TF to PyTorch. @@ -438,12 +420,8 @@ class MobileNetV2Stem(nn.Module): return features +@auto_docstring class MobileNetV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MobileNetV2Config load_tf_weights = load_tf_weights_in_mobilenet_v2 base_model_prefix = "mobilenet_v2" @@ -462,36 +440,13 @@ class MobileNetV2PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MOBILENET_V2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MobileNetV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MOBILENET_V2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`MobileNetV2ImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MobileNetV2 model outputting raw hidden-states without any specific head on top.", - MOBILENET_V2_START_DOCSTRING, -) +@auto_docstring class MobileNetV2Model(MobileNetV2PreTrainedModel): def __init__(self, config: MobileNetV2Config, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -554,14 +509,7 @@ class MobileNetV2Model(MobileNetV2PreTrainedModel): def _prune_heads(self, heads_to_prune): raise NotImplementedError - @add_start_docstrings_to_model_forward(MOBILENET_V2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -603,12 +551,11 @@ class MobileNetV2Model(MobileNetV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileNetV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - MOBILENET_V2_START_DOCSTRING, + """ ) class MobileNetV2ForImageClassification(MobileNetV2PreTrainedModel): def __init__(self, config: MobileNetV2Config) -> None: @@ -626,13 +573,7 @@ class MobileNetV2ForImageClassification(MobileNetV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILENET_V2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -763,11 +704,10 @@ class MobileNetV2DeepLabV3Plus(nn.Module): return features -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileNetV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. - """, - MOBILENET_V2_START_DOCSTRING, + """ ) class MobileNetV2ForSemanticSegmentation(MobileNetV2PreTrainedModel): def __init__(self, config: MobileNetV2Config) -> None: @@ -780,8 +720,7 @@ class MobileNetV2ForSemanticSegmentation(MobileNetV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILENET_V2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -794,8 +733,6 @@ class MobileNetV2ForSemanticSegmentation(MobileNetV2PreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/mobilevit/modeling_mobilevit.py b/src/transformers/models/mobilevit/modeling_mobilevit.py index f41da2bafa..b642f5c12e 100755 --- a/src/transformers/models/mobilevit/modeling_mobilevit.py +++ b/src/transformers/models/mobilevit/modeling_mobilevit.py @@ -33,32 +33,13 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_mobilevit import MobileViTConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "MobileViTConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "apple/mobilevit-small" -_EXPECTED_OUTPUT_SHAPE = [1, 640, 8, 8] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "apple/mobilevit-small" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - - def make_divisible(value: int, divisor: int = 8, min_value: Optional[int] = None) -> int: """ Ensure that all layers have a channel count that is divisible by `divisor`. This function is taken from the @@ -639,12 +620,8 @@ class MobileViTEncoder(nn.Module): return BaseModelOutputWithNoAttention(last_hidden_state=hidden_states, hidden_states=all_hidden_states) +@auto_docstring class MobileViTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MobileViTConfig base_model_prefix = "mobilevit" main_input_name = "pixel_values" @@ -664,36 +641,14 @@ class MobileViTPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MOBILEVIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MobileViTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MOBILEVIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`MobileViTImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MobileViT model outputting raw hidden-states without any specific head on top.", - MOBILEVIT_START_DOCSTRING, -) +@auto_docstring class MobileViTModel(MobileViTPreTrainedModel): def __init__(self, config: MobileViTConfig, expand_output: bool = True): + r""" + expand_output (`bool`, *optional*, defaults to `True`): + Whether to expand the output of the model using a 1x1 convolution. If `True`, the model will apply an additional + 1x1 convolution to expand the output channels from `config.neck_hidden_sizes[5]` to `config.neck_hidden_sizes[6]`. + """ super().__init__(config) self.config = config self.expand_output = expand_output @@ -729,14 +684,7 @@ class MobileViTModel(MobileViTPreTrainedModel): for transformer_layer in mobilevit_layer.transformer.layer: transformer_layer.attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MOBILEVIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -779,12 +727,11 @@ class MobileViTModel(MobileViTPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileViT model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - MOBILEVIT_START_DOCSTRING, + """ ) class MobileViTForImageClassification(MobileViTPreTrainedModel): def __init__(self, config: MobileViTConfig) -> None: @@ -802,13 +749,7 @@ class MobileViTForImageClassification(MobileViTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEVIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -975,11 +916,10 @@ class MobileViTDeepLabV3(nn.Module): return features -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileViT model with a semantic segmentation head on top, e.g. for Pascal VOC. - """, - MOBILEVIT_START_DOCSTRING, + """ ) class MobileViTForSemanticSegmentation(MobileViTPreTrainedModel): def __init__(self, config: MobileViTConfig) -> None: @@ -992,8 +932,7 @@ class MobileViTForSemanticSegmentation(MobileViTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEVIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1006,8 +945,6 @@ class MobileViTForSemanticSegmentation(MobileViTPreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/mobilevitv2/modeling_mobilevitv2.py b/src/transformers/models/mobilevitv2/modeling_mobilevitv2.py index 5848aada8b..7f800bfd34 100644 --- a/src/transformers/models/mobilevitv2/modeling_mobilevitv2.py +++ b/src/transformers/models/mobilevitv2/modeling_mobilevitv2.py @@ -31,31 +31,13 @@ from ...modeling_outputs import ( SemanticSegmenterOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_mobilevitv2 import MobileViTV2Config logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "MobileViTV2Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "apple/mobilevitv2-1.0-imagenet1k-256" -_EXPECTED_OUTPUT_SHAPE = [1, 512, 8, 8] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "apple/mobilevitv2-1.0-imagenet1k-256" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - - # Copied from transformers.models.mobilevit.modeling_mobilevit.make_divisible def make_divisible(value: int, divisor: int = 8, min_value: Optional[int] = None) -> int: """ @@ -591,13 +573,9 @@ class MobileViTV2Encoder(nn.Module): return BaseModelOutputWithNoAttention(last_hidden_state=hidden_states, hidden_states=all_hidden_states) +@auto_docstring # Copied from transformers.models.mobilevit.modeling_mobilevit.MobileViTPreTrainedModel with MobileViT->MobileViTV2,mobilevit->mobilevitv2 class MobileViTV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MobileViTV2Config base_model_prefix = "mobilevitv2" main_input_name = "pixel_values" @@ -617,36 +595,14 @@ class MobileViTV2PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MOBILEVITV2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MobileViTV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MOBILEVITV2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`MobileViTImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MobileViTV2 model outputting raw hidden-states without any specific head on top.", - MOBILEVITV2_START_DOCSTRING, -) +@auto_docstring class MobileViTV2Model(MobileViTV2PreTrainedModel): def __init__(self, config: MobileViTV2Config, expand_output: bool = True): + r""" + expand_output (`bool`, *optional*, defaults to `True`): + Whether to expand the output of the model. If `True`, the model will output pooled features in addition to + hidden states. If `False`, only the hidden states will be returned. + """ super().__init__(config) self.config = config self.expand_output = expand_output @@ -679,14 +635,7 @@ class MobileViTV2Model(MobileViTV2PreTrainedModel): for transformer_layer in mobilevitv2_layer.transformer.layer: transformer_layer.attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MOBILEVITV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -729,12 +678,11 @@ class MobileViTV2Model(MobileViTV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - MOBILEVITV2_START_DOCSTRING, + """ ) class MobileViTV2ForImageClassification(MobileViTV2PreTrainedModel): def __init__(self, config: MobileViTV2Config) -> None: @@ -754,13 +702,7 @@ class MobileViTV2ForImageClassification(MobileViTV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEVITV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -930,11 +872,10 @@ class MobileViTV2DeepLabV3(nn.Module): return features -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. - """, - MOBILEVITV2_START_DOCSTRING, + """ ) class MobileViTV2ForSemanticSegmentation(MobileViTV2PreTrainedModel): def __init__(self, config: MobileViTV2Config) -> None: @@ -947,8 +888,7 @@ class MobileViTV2ForSemanticSegmentation(MobileViTV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MOBILEVITV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -961,8 +901,6 @@ class MobileViTV2ForSemanticSegmentation(MobileViTV2PreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/modernbert/modeling_modernbert.py b/src/transformers/models/modernbert/modeling_modernbert.py index 974a4dc1e4..c0e9909715 100644 --- a/src/transformers/models/modernbert/modeling_modernbert.py +++ b/src/transformers/models/modernbert/modeling_modernbert.py @@ -39,13 +39,7 @@ from ...modeling_outputs import ( ) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_flash_attn_2_available, - logging, -) +from ...utils import auto_docstring, is_flash_attn_2_available, logging from ...utils.import_utils import is_triton_available from .configuration_modernbert import ModernBertConfig @@ -57,10 +51,8 @@ if is_flash_attn_2_available(): else: RotaryEmbedding = object -logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "answerdotai/ModernBERT-base" -_CONFIG_FOR_DOC = "ModernBertConfig" +logger = logging.get_logger(__name__) class ApplyRotaryEmbUnpad(torch.autograd.Function): @@ -562,27 +554,7 @@ class ModernBertEncoderLayer(nn.Module): return (hidden_states,) + attn_outputs[1:] # add attentions if outputted -MODERNBERT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ModernBertConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare ModernBert Model outputting raw hidden-states without any specific head on top.", - MODERNBERT_START_DOCSTRING, -) +@auto_docstring class ModernBertPreTrainedModel(PreTrainedModel): config_class = ModernBertConfig base_model_prefix = "model" @@ -788,71 +760,7 @@ def _pad_modernbert_output( return padded_inputs -MODERNBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. With Flash Attention 2.0, padding will be ignored - by default should you provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers - perform global attention, while the rest perform local attention. This mask is used to avoid attending to - far-away tokens in the local attention layers when not using Flash Attention. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): - Indices of the non-padding tokens in the input sequence. Used for unpadding the output. - cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): - Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. - max_seqlen (`int`, *optional*): - Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. - batch_size (`int`, *optional*): - Batch size of the input sequences. Used to pad the output tensors. - seq_len (`int`, *optional*): - Sequence length of the input sequences including padding tokens. Used to pad the output tensors. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ModernBert Model outputting raw hidden-states without any specific head on top.", - MODERNBERT_START_DOCSTRING, -) +@auto_docstring class ModernBertModel(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): super().__init__(config) @@ -871,12 +779,7 @@ class ModernBertModel(ModernBertPreTrainedModel): def set_input_embeddings(self, value): self.embeddings.tok_embeddings = value - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -893,6 +796,22 @@ class ModernBertModel(ModernBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutput]: + r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1039,9 +958,10 @@ class ModernBertPredictionHead(nn.Module): return self.norm(self.act(self.dense(hidden_states))) -@add_start_docstrings( - "The ModernBert Model with a decoder head on top that is used for masked language modeling.", - MODERNBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ModernBert Model with a decoder head on top that is used for masked language modeling. + """ ) class ModernBertForMaskedLM(ModernBertPreTrainedModel): _tied_weights_keys = ["decoder.weight"] @@ -1069,12 +989,7 @@ class ModernBertForMaskedLM(ModernBertPreTrainedModel): def compiled_head(self, output: torch.Tensor) -> torch.Tensor: return self.decoder(self.head(output)) - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1093,6 +1008,22 @@ class ModernBertForMaskedLM(ModernBertPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: + r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() @@ -1171,9 +1102,10 @@ class ModernBertForMaskedLM(ModernBertPreTrainedModel): ) -@add_start_docstrings( - "The ModernBert Model with a sequence classification head on top that performs pooling.", - MODERNBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ModernBert Model with a sequence classification head on top that performs pooling. + """ ) class ModernBertForSequenceClassification(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): @@ -1189,12 +1121,7 @@ class ModernBertForSequenceClassification(ModernBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1214,10 +1141,24 @@ class ModernBertForSequenceClassification(ModernBertPreTrainedModel): **kwargs, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() @@ -1285,9 +1226,10 @@ class ModernBertForSequenceClassification(ModernBertPreTrainedModel): ) -@add_start_docstrings( - "The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.", - MODERNBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks. + """ ) class ModernBertForTokenClassification(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): @@ -1302,12 +1244,7 @@ class ModernBertForTokenClassification(ModernBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1326,8 +1263,22 @@ class ModernBertForTokenClassification(ModernBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() @@ -1370,13 +1321,7 @@ class ModernBertForTokenClassification(ModernBertPreTrainedModel): ) -@add_start_docstrings( - """ - The ModernBert Model with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MODERNBERT_START_DOCSTRING, -) +@auto_docstring class ModernBertForQuestionAnswering(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): super().__init__(config) @@ -1389,12 +1334,7 @@ class ModernBertForQuestionAnswering(ModernBertPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor], @@ -1413,6 +1353,22 @@ class ModernBertForQuestionAnswering(ModernBertPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: + r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() diff --git a/src/transformers/models/modernbert/modular_modernbert.py b/src/transformers/models/modernbert/modular_modernbert.py index 9e74b4d787..137673cfa5 100644 --- a/src/transformers/models/modernbert/modular_modernbert.py +++ b/src/transformers/models/modernbert/modular_modernbert.py @@ -34,13 +34,7 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_flash_attn_2_available, - logging, -) +from ...utils import auto_docstring, is_flash_attn_2_available, logging from ...utils.import_utils import is_triton_available from ..gemma.modeling_gemma import GemmaRotaryEmbedding, apply_rotary_pos_emb @@ -52,8 +46,6 @@ if is_flash_attn_2_available(): else: RotaryEmbedding = object -_CHECKPOINT_FOR_DOC = "answerdotai/ModernBERT-base" -_CONFIG_FOR_DOC = "ModernBertConfig" logger = logging.get_logger(__name__) @@ -764,27 +756,7 @@ class ModernBertEncoderLayer(nn.Module): return (hidden_states,) + attn_outputs[1:] # add attentions if outputted -MODERNBERT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ModernBertConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare ModernBert Model outputting raw hidden-states without any specific head on top.", - MODERNBERT_START_DOCSTRING, -) +@auto_docstring class ModernBertPreTrainedModel(PreTrainedModel): config_class = ModernBertConfig base_model_prefix = "model" @@ -918,71 +890,7 @@ class ModernBertPreTrainedModel(PreTrainedModel): return model_embeds -MODERNBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. With Flash Attention 2.0, padding will be ignored - by default should you provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers - perform global attention, while the rest perform local attention. This mask is used to avoid attending to - far-away tokens in the local attention layers when not using Flash Attention. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): - Indices of the non-padding tokens in the input sequence. Used for unpadding the output. - cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): - Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. - max_seqlen (`int`, *optional*): - Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. - batch_size (`int`, *optional*): - Batch size of the input sequences. Used to pad the output tensors. - seq_len (`int`, *optional*): - Sequence length of the input sequences including padding tokens. Used to pad the output tensors. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ModernBert Model outputting raw hidden-states without any specific head on top.", - MODERNBERT_START_DOCSTRING, -) +@auto_docstring class ModernBertModel(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): super().__init__(config) @@ -1001,12 +909,7 @@ class ModernBertModel(ModernBertPreTrainedModel): def set_input_embeddings(self, value): self.embeddings.tok_embeddings = value - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1023,6 +926,22 @@ class ModernBertModel(ModernBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutput]: + r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1169,9 +1088,10 @@ class ModernBertPredictionHead(nn.Module): return self.norm(self.act(self.dense(hidden_states))) -@add_start_docstrings( - "The ModernBert Model with a decoder head on top that is used for masked language modeling.", - MODERNBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ModernBert Model with a decoder head on top that is used for masked language modeling. + """ ) class ModernBertForMaskedLM(ModernBertPreTrainedModel): _tied_weights_keys = ["decoder.weight"] @@ -1199,12 +1119,7 @@ class ModernBertForMaskedLM(ModernBertPreTrainedModel): def compiled_head(self, output: torch.Tensor) -> torch.Tensor: return self.decoder(self.head(output)) - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1223,6 +1138,22 @@ class ModernBertForMaskedLM(ModernBertPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: + r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() @@ -1301,9 +1232,10 @@ class ModernBertForMaskedLM(ModernBertPreTrainedModel): ) -@add_start_docstrings( - "The ModernBert Model with a sequence classification head on top that performs pooling.", - MODERNBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ModernBert Model with a sequence classification head on top that performs pooling. + """ ) class ModernBertForSequenceClassification(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): @@ -1319,12 +1251,7 @@ class ModernBertForSequenceClassification(ModernBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1344,10 +1271,24 @@ class ModernBertForSequenceClassification(ModernBertPreTrainedModel): **kwargs, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() @@ -1415,9 +1356,10 @@ class ModernBertForSequenceClassification(ModernBertPreTrainedModel): ) -@add_start_docstrings( - "The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.", - MODERNBERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks. + """ ) class ModernBertForTokenClassification(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): @@ -1432,12 +1374,7 @@ class ModernBertForTokenClassification(ModernBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1456,8 +1393,22 @@ class ModernBertForTokenClassification(ModernBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() @@ -1500,13 +1451,7 @@ class ModernBertForTokenClassification(ModernBertPreTrainedModel): ) -@add_start_docstrings( - """ - The ModernBert Model with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MODERNBERT_START_DOCSTRING, -) +@auto_docstring class ModernBertForQuestionAnswering(ModernBertPreTrainedModel): def __init__(self, config: ModernBertConfig): super().__init__(config) @@ -1519,12 +1464,7 @@ class ModernBertForQuestionAnswering(ModernBertPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(MODERNBERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor], @@ -1543,6 +1483,22 @@ class ModernBertForQuestionAnswering(ModernBertPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: + r""" + sliding_window_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding or far-away tokens. In ModernBert, only every few layers + perform global attention, while the rest perform local attention. This mask is used to avoid attending to + far-away tokens in the local attention layers when not using Flash Attention. + indices (`torch.Tensor` of shape `(total_unpadded_tokens,)`, *optional*): + Indices of the non-padding tokens in the input sequence. Used for unpadding the output. + cu_seqlens (`torch.Tensor` of shape `(batch + 1,)`, *optional*): + Cumulative sequence lengths of the input sequences. Used to index the unpadded tensors. + max_seqlen (`int`, *optional*): + Maximum sequence length in the batch excluding padding tokens. Used to unpad input_ids and pad output tensors. + batch_size (`int`, *optional*): + Batch size of the input sequences. Used to pad the output tensors. + seq_len (`int`, *optional*): + Sequence length of the input sequences including padding tokens. Used to pad the output tensors. + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict self._maybe_set_compile() diff --git a/src/transformers/models/moonshine/modeling_moonshine.py b/src/transformers/models/moonshine/modeling_moonshine.py index 475b24d907..d849d1945b 100644 --- a/src/transformers/models/moonshine/modeling_moonshine.py +++ b/src/transformers/models/moonshine/modeling_moonshine.py @@ -44,14 +44,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_moonshine import MoonshineConfig @@ -62,7 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MoonshineConfig" class MoonshineEncoderMLP(nn.Module): @@ -498,27 +490,7 @@ class MoonshineDecoderLayer(GradientCheckpointingLayer): return outputs -MOONSHINE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MoonshineConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Moonshine Model outputting raw hidden-states without any specific head on top.", - MOONSHINE_START_DOCSTRING, -) +@auto_docstring class MoonshinePreTrainedModel(PreTrainedModel): config_class = MoonshineConfig base_model_prefix = "model" @@ -695,88 +667,8 @@ class MoonshineEncoder(MoonshinePreTrainedModel): ) -MOONSHINE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Moonshine Model outputting raw hidden-states without any specific head on top.", - MOONSHINE_START_DOCSTRING, -) +@auto_docstring class MoonshineDecoder(MoonshinePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineDecoderLayer`] - - Args: - config: MoonshineConfig - """ - main_input_name = "input_ids" def __init__(self, config: MoonshineConfig): @@ -802,7 +694,7 @@ class MoonshineDecoder(MoonshinePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(MOONSHINE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -818,16 +710,15 @@ class MoonshineDecoder(MoonshinePreTrainedModel): encoder_attention_mask: Optional[torch.Tensor] = None, **flash_attn_kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, BaseModelOutputWithPast]: - """ - Args: - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - of the decoder. - encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + [What are attention masks?](../glossary#attention-mask) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1178,100 +1069,7 @@ def _compute_mask_indices( return spec_aug_mask -MOONSHINE_MODEL_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): - Float values of the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_values`, the [`AutoFeatureExtractor`] should be used for padding - and conversion into a tensor of type `torch.FloatTensor`. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - decoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` - of shape `(batch_size, sequence_length)`. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `decoder_position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Moonshine Model outputting raw hidden-states without any specific head on top.", - MOONSHINE_START_DOCSTRING, -) +@auto_docstring class MoonshineModel(MoonshinePreTrainedModel): def __init__(self, config: MoonshineConfig): super().__init__(config) @@ -1344,8 +1142,7 @@ class MoonshineModel(MoonshinePreTrainedModel): return input_features @can_return_tuple - @add_start_docstrings_to_model_forward(MOONSHINE_MODEL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.FloatTensor] = None, @@ -1362,7 +1159,49 @@ class MoonshineModel(MoonshinePreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Seq2SeqModelOutput: r""" - Returns: + input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): + Float values of the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_values`, the [`AutoFeatureExtractor`] should be used for padding + and conversion into a tensor of type `torch.FloatTensor`. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) Example: @@ -1380,7 +1219,8 @@ class MoonshineModel(MoonshinePreTrainedModel): >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state >>> list(last_hidden_state.shape) [1, 2, 288] - ```""" + ``` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1445,9 +1285,10 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start return shifted_input_ids -@add_start_docstrings( - "The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.", - MOONSHINE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Moonshine Model with a language modeling head. Can be used for automatic speech recognition. + """ ) class MoonshineForConditionalGeneration(MoonshinePreTrainedModel, GenerationMixin): _tied_weights_keys = ["proj_out.weight"] @@ -1476,8 +1317,7 @@ class MoonshineForConditionalGeneration(MoonshinePreTrainedModel, GenerationMixi return self.model.get_input_embeddings() @can_return_tuple - @add_start_docstrings_to_model_forward(MOONSHINE_MODEL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.FloatTensor] = None, @@ -1495,13 +1335,54 @@ class MoonshineForConditionalGeneration(MoonshinePreTrainedModel, GenerationMixi labels: Optional[torch.LongTensor] = None, ) -> Seq2SeqLMOutput: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): + Float values of the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_values`, the [`AutoFeatureExtractor`] should be used for padding + and conversion into a tensor of type `torch.FloatTensor`. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: - Example: ```python diff --git a/src/transformers/models/moonshine/modular_moonshine.py b/src/transformers/models/moonshine/modular_moonshine.py index 8864c0b41b..edb1a70279 100644 --- a/src/transformers/models/moonshine/modular_moonshine.py +++ b/src/transformers/models/moonshine/modular_moonshine.py @@ -21,10 +21,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache from ...configuration_utils import PretrainedConfig from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - _prepare_4d_attention_mask, - _prepare_4d_attention_mask_for_sdpa, -) +from ...modeling_attn_mask_utils import _prepare_4d_attention_mask, _prepare_4d_attention_mask_for_sdpa from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( @@ -37,13 +34,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import rope_config_validation from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, logging from ..glm.modeling_glm import GlmAttention, GlmRotaryEmbedding, apply_rotary_pos_emb from ..llama.modeling_llama import LlamaDecoderLayer, LlamaModel, eager_attention_forward from ..whisper.modeling_whisper import WhisperModel, shift_tokens_right @@ -51,9 +42,6 @@ from ..whisper.modeling_whisper import WhisperModel, shift_tokens_right logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "UsefulSensors/moonshine-tiny" -_CONFIG_FOR_DOC = "MoonshineConfig" - class MoonshineConfig(PretrainedConfig): r""" @@ -516,27 +504,7 @@ class MoonshineDecoderLayer(GradientCheckpointingLayer): return outputs -MOONSHINE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MoonshineConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Moonshine Model outputting raw hidden-states without any specific head on top.", - MOONSHINE_START_DOCSTRING, -) +@auto_docstring class MoonshinePreTrainedModel(PreTrainedModel): config_class = MoonshineConfig base_model_prefix = "model" @@ -738,16 +706,15 @@ class MoonshineDecoder(LlamaModel): encoder_attention_mask: Optional[torch.Tensor] = None, **flash_attn_kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, BaseModelOutputWithPast]: - """ - Args: - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - of the decoder. - encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + [What are attention masks?](../glossary#attention-mask) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -856,19 +823,31 @@ class MoonshineDecoder(LlamaModel): ) -MOONSHINE_MODEL_INPUTS_DOCSTRING = r""" - Args: +class MoonshineModel(WhisperModel): + @can_return_tuple + @auto_docstring + def forward( + self, + input_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Union[EncoderDecoderCache, Tuple[torch.FloatTensor]]] = None, + decoder_inputs_embeds: Optional[Tuple[torch.FloatTensor]] = None, + decoder_position_ids: Optional[Tuple[torch.LongTensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Seq2SeqModelOutput: + r""" input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): Float values of the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoFeatureExtractor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. @@ -897,28 +876,6 @@ MOONSHINE_MODEL_INPUTS_DOCSTRING = r""" - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` - of shape `(batch_size, sequence_length)`. decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the @@ -928,48 +885,9 @@ MOONSHINE_MODEL_INPUTS_DOCSTRING = r""" config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `decoder_position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" + Example: -@add_start_docstrings( - "The bare Moonshine Model outputting raw hidden-states without any specific head on top.", - MOONSHINE_START_DOCSTRING, -) -class MoonshineModel(WhisperModel): - @can_return_tuple - @add_start_docstrings_to_model_forward(MOONSHINE_MODEL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) - def forward( - self, - input_values: Optional[torch.FloatTensor] = None, - attention_mask: Optional[torch.LongTensor] = None, - decoder_input_ids: Optional[torch.LongTensor] = None, - decoder_attention_mask: Optional[torch.LongTensor] = None, - encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, - past_key_values: Optional[Union[EncoderDecoderCache, Tuple[torch.FloatTensor]]] = None, - decoder_inputs_embeds: Optional[Tuple[torch.FloatTensor]] = None, - decoder_position_ids: Optional[Tuple[torch.LongTensor]] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - cache_position: Optional[torch.LongTensor] = None, - ) -> Seq2SeqModelOutput: - r""" ```python >>> import torch >>> from transformers import AutoFeatureExtractor, MoonshineModel @@ -984,7 +902,8 @@ class MoonshineModel(WhisperModel): >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state >>> list(last_hidden_state.shape) [1, 2, 288] - ```""" + ``` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1033,9 +952,10 @@ class MoonshineModel(WhisperModel): ) -@add_start_docstrings( - "The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.", - MOONSHINE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Moonshine Model with a language modeling head. Can be used for automatic speech recognition. + """ ) class MoonshineForConditionalGeneration(MoonshinePreTrainedModel, GenerationMixin): _tied_weights_keys = ["proj_out.weight"] @@ -1064,8 +984,7 @@ class MoonshineForConditionalGeneration(MoonshinePreTrainedModel, GenerationMixi return self.model.get_input_embeddings() @can_return_tuple - @add_start_docstrings_to_model_forward(MOONSHINE_MODEL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.FloatTensor] = None, @@ -1083,13 +1002,54 @@ class MoonshineForConditionalGeneration(MoonshinePreTrainedModel, GenerationMixi labels: Optional[torch.LongTensor] = None, ) -> Seq2SeqLMOutput: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`): + Float values of the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_values`, the [`AutoFeatureExtractor`] should be used for padding + and conversion into a tensor of type `torch.FloatTensor`. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + decoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: - Example: ```python diff --git a/src/transformers/models/moshi/modeling_moshi.py b/src/transformers/models/moshi/modeling_moshi.py index 4bc31452a3..ef705af416 100644 --- a/src/transformers/models/moshi/modeling_moshi.py +++ b/src/transformers/models/moshi/modeling_moshi.py @@ -25,29 +25,14 @@ from torch.nn import CrossEntropyLoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache, SlidingWindowCache, StaticCache -from ...generation import ( - GenerationConfig, - GenerationMixin, -) +from ...generation import GenerationConfig, GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available -from ...modeling_outputs import ( - BaseModelOutputWithPast, - CausalLMOutputWithPast, - ModelOutput, - Seq2SeqLMOutput, -) +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, ModelOutput, Seq2SeqLMOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, is_torchdynamo_compiling, logging from ..auto.modeling_auto import AutoModel from .configuration_moshi import MoshiConfig, MoshiDepthConfig @@ -63,9 +48,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MoshiConfig" -_CHECKPOINT_FOR_DOC = "kmhf/hf-moshiko" - @dataclass class MoshiConditionalGenerationGenerateOutput(ModelOutput): @@ -838,12 +820,8 @@ class MoshiDecoderLayer(nn.Module): return outputs +@auto_docstring class MoshiPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MoshiConfig base_model_prefix = "model" supports_gradient_checkpointing = True @@ -870,164 +848,6 @@ class MoshiPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MOSHI_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MoshiConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -MOSHI_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence text tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - user_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*): - The audio waveforms used as audio user prompt for the generation. - user_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*): - The audio codes used as audio user prompt for the generation. Has priority over `user_input_values` and represents the audio "tokens" of `user_input_values` once passed through the audio encoder. - moshi_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*): - The audio waveforms used as audio Moshi prompt for the generation. - moshi_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*): - The audio codes used as audio Moshi prompt for the generation. Has priority over `moshi_input_values` and represents the audio "tokens" of `moshi_input_values` once passed through the audio encoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `input_ids` and `inputs_embeds` are both unset, `inputs_embeds` takes the value - of `inputs_embeds`. - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - text_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for text language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - audio_labels (`torch.LongTensor` of shape `(batch_size, num_codebooks, sequence_length)`, *optional*): - Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.audio_vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -MOSHI_DECODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class MoshiDepthDecoder(MoshiPreTrainedModel, GenerationMixin): """ Transformer depth decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoshiTransformerLayer`] @@ -1421,18 +1241,8 @@ class MoshiDepthDecoder(MoshiPreTrainedModel, GenerationMixin): return causal_mask -@add_start_docstrings( - "The bare Moshi Model outputting raw hidden-states without any specific head on top.", - MOSHI_START_DOCSTRING, -) +@auto_docstring class MoshiModel(MoshiPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoshiDecoderLayer`] - - Args: - config: MoshiConfig - """ - def __init__(self, config: MoshiConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1457,7 +1267,7 @@ class MoshiModel(MoshiPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(MOSHI_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1737,9 +1547,10 @@ class MoshiModel(MoshiPreTrainedModel): return causal_mask -@add_start_docstrings( - "The Moshi decoder model with a text language modelling head on top. Only usable for text.", - MOSHI_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Moshi decoder model with a text language modelling head on top. Only usable for text. + """ ) class MoshiForCausalLM(MoshiPreTrainedModel, GenerationMixin): _tied_weights_keys = ["model.embed_tokens.weight", "lm_head.weight"] @@ -1772,8 +1583,7 @@ class MoshiForCausalLM(MoshiPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(MOSHI_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoshiCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1791,19 +1601,10 @@ class MoshiForCausalLM(MoshiPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, MoshiCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1886,9 +1687,10 @@ class MoshiForCausalLM(MoshiPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - "The original Moshi model with an audio encoder, a Moshi depth decoder and a Moshi decoder, for speech-to-speech.", - MOSHI_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The original Moshi model with an audio encoder, a Moshi depth decoder and a Moshi decoder, for speech-to-speech. + """ ) class MoshiForConditionalGeneration(MoshiPreTrainedModel, GenerationMixin): _tied_weights_keys = ["decoder.model.embed_tokens.weight", "decoder.lm_head.weight"] @@ -1921,8 +1723,7 @@ class MoshiForConditionalGeneration(MoshiPreTrainedModel, GenerationMixin): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(MOSHI_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1942,7 +1743,30 @@ class MoshiForConditionalGeneration(MoshiPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" - Returns: + user_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*): + The audio waveforms used as audio user prompt for the generation. + user_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*): + The audio codes used as audio user prompt for the generation. Has priority over `user_input_values` and represents the audio "tokens" of `user_input_values` once passed through the audio encoder. + moshi_input_values (`torch.Tensor `of shape `(batch_size, 1, audio_sequence_length), *optional*): + The audio waveforms used as audio Moshi prompt for the generation. + moshi_audio_codes (`torch.Tensor `of shape `(batch_size, num_codebooks, sequence_length), *optional*): + The audio codes used as audio Moshi prompt for the generation. Has priority over `moshi_input_values` and represents the audio "tokens" of `moshi_input_values` once passed through the audio encoder. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + + If `input_ids` and `inputs_embeds` are both unset, `inputs_embeds` takes the value + of `inputs_embeds`. + text_labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for text language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + audio_labels (`torch.LongTensor` of shape `(batch_size, num_codebooks, sequence_length)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.audio_vocab_size]` Examples: ```python diff --git a/src/transformers/models/mpnet/modeling_mpnet.py b/src/transformers/models/mpnet/modeling_mpnet.py index 848f1c1703..1e1a18c559 100644 --- a/src/transformers/models/mpnet/modeling_mpnet.py +++ b/src/transformers/models/mpnet/modeling_mpnet.py @@ -34,16 +34,14 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_mpnet import MPNetConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/mpnet-base" -_CONFIG_FOR_DOC = "MPNetConfig" - +@auto_docstring class MPNetPreTrainedModel(PreTrainedModel): config_class = MPNetConfig base_model_prefix = "mpnet" @@ -413,70 +411,13 @@ class MPNetPooler(nn.Module): return pooled_output -MPNET_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MPNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MPNET_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MPNet Model transformer outputting raw hidden-states without any specific head on top.", - MPNET_START_DOCSTRING, -) +@auto_docstring class MPNetModel(MPNetPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -501,12 +442,7 @@ class MPNetModel(MPNetPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -584,12 +520,7 @@ class MPNetForMaskedLM(MPNetPreTrainedModel): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -669,12 +600,11 @@ class MPNetLMHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MPNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MPNET_START_DOCSTRING, + """ ) class MPNetForSequenceClassification(MPNetPreTrainedModel): def __init__(self, config): @@ -687,12 +617,7 @@ class MPNetForSequenceClassification(MPNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -761,13 +686,7 @@ class MPNetForSequenceClassification(MPNetPreTrainedModel): ) -@add_start_docstrings( - """ - MPNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - MPNET_START_DOCSTRING, -) +@auto_docstring class MPNetForMultipleChoice(MPNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -779,12 +698,7 @@ class MPNetForMultipleChoice(MPNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -798,6 +712,22 @@ class MPNetForMultipleChoice(MPNetPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -849,13 +779,7 @@ class MPNetForMultipleChoice(MPNetPreTrainedModel): ) -@add_start_docstrings( - """ - MPNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - MPNET_START_DOCSTRING, -) +@auto_docstring class MPNetForTokenClassification(MPNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -868,12 +792,7 @@ class MPNetForTokenClassification(MPNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -945,13 +864,7 @@ class MPNetClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - MPNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MPNET_START_DOCSTRING, -) +@auto_docstring class MPNetForQuestionAnswering(MPNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -963,12 +876,7 @@ class MPNetForQuestionAnswering(MPNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -982,17 +890,6 @@ class MPNetForQuestionAnswering(MPNetPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.mpnet( diff --git a/src/transformers/models/mpt/modeling_mpt.py b/src/transformers/models/mpt/modeling_mpt.py index bff4c0b06c..10e91c988a 100644 --- a/src/transformers/models/mpt/modeling_mpt.py +++ b/src/transformers/models/mpt/modeling_mpt.py @@ -23,7 +23,6 @@ from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss from torch.nn import functional as F -from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask from ...modeling_outputs import ( @@ -34,15 +33,12 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import logging +from ...utils import auto_docstring, logging from .configuration_mpt import MptConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "mosaicml/mpt-7b" -_CONFIG_FOR_DOC = "MptConfig" - def build_mpt_alibi_tensor(num_heads, sequence_length, alibi_bias_max=8, device=None): r""" @@ -228,6 +224,7 @@ class MptBlock(nn.Module): return outputs # hidden_states, present, attentions +@auto_docstring class MptPreTrainedModel(PreTrainedModel): config_class = MptConfig base_model_prefix = "transformer" @@ -275,75 +272,7 @@ class MptPreTrainedModel(PreTrainedModel): ) -MPT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MptConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` - (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`): - Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see - `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have - their past given to this model should not be passed as `input_ids` as they have already been computed. - - Each element of `past_key_values` is a tuple (past_key, past_value): - - past_key: [batch_size * num_heads, head_dim, kv_length] - - past_value: [batch_size * num_heads, kv_length, head_dim] - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see - `past_key_values`). - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Mpt Model transformer outputting raw hidden-states without any specific head on top.", - MPT_START_DOCSTRING, -) +@auto_docstring class MptModel(MptPreTrainedModel): def __init__(self, config: MptConfig): super().__init__(config) @@ -376,12 +305,7 @@ class MptModel(MptPreTrainedModel): def set_input_embeddings(self, new_embeddings: torch.Tensor): self.wte = new_embeddings - @add_start_docstrings_to_model_forward(MPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -394,6 +318,19 @@ class MptModel(MptPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, # NOOP kwargs, for now ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -495,12 +432,11 @@ class MptModel(MptPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The MPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - MPT_START_DOCSTRING, + """ ) class MptForCausalLM(MptPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -519,12 +455,7 @@ class MptForCausalLM(MptPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings: torch.Tensor): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(MPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -539,6 +470,17 @@ class MptForCausalLM(MptPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -608,8 +550,8 @@ class MptForCausalLM(MptPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The MPT Model transformer with a sequence classification head on top (linear layer). [`MptForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -620,8 +562,7 @@ class MptForCausalLM(MptPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - MPT_START_DOCSTRING, + """ ) class MptForSequenceClassification(MptPreTrainedModel): def __init__(self, config: MptConfig): @@ -633,12 +574,7 @@ class MptForSequenceClassification(MptPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -652,6 +588,17 @@ class MptForSequenceClassification(MptPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -731,13 +678,7 @@ class MptForSequenceClassification(MptPreTrainedModel): ) -@add_start_docstrings( - """ - MPT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - MPT_START_DOCSTRING, -) +@auto_docstring class MptForTokenClassification(MptPreTrainedModel): def __init__(self, config: MptConfig): super().__init__(config) @@ -756,12 +697,7 @@ class MptForTokenClassification(MptPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -776,6 +712,17 @@ class MptForTokenClassification(MptPreTrainedModel): **deprecated_arguments, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -820,13 +767,7 @@ class MptForTokenClassification(MptPreTrainedModel): ) -@add_start_docstrings( - """ - The MPT Model transformer with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MPT_START_DOCSTRING, -) +@auto_docstring class MptForQuestionAnswering(MptPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -836,7 +777,7 @@ class MptForQuestionAnswering(MptPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -849,14 +790,17 @@ class MptForQuestionAnswering(MptPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]` + (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/mra/modeling_mra.py b/src/transformers/models/mra/modeling_mra.py index 6071d6a9e9..615df3d7d0 100644 --- a/src/transformers/models/mra/modeling_mra.py +++ b/src/transformers/models/mra/modeling_mra.py @@ -35,24 +35,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_ninja_available, - is_torch_cuda_available, - logging, -) +from ...utils import auto_docstring, is_ninja_available, is_torch_cuda_available, logging from .configuration_mra import MraConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "uw-madison/mra-base-512-4" -_CONFIG_FOR_DOC = "MraConfig" -_TOKENIZER_FOR_DOC = "AutoTokenizer" - - mra_cuda_kernel = None @@ -825,13 +813,9 @@ class MraOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring # Copied from transformers.models.yoso.modeling_yoso.YosoPreTrainedModel with Yoso->Mra,yoso->mra class MraPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MraConfig base_model_prefix = "mra" supports_gradient_checkpointing = True @@ -853,68 +837,7 @@ class MraPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -MRA_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`MraConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MRA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare MRA Model transformer outputting raw hidden-states without any specific head on top.", - MRA_START_DOCSTRING, -) +@auto_docstring class MraModel(MraPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -940,12 +863,7 @@ class MraModel(MraPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1023,7 +941,7 @@ class MraModel(MraPreTrainedModel): ) -@add_start_docstrings("""MRA Model with a `language modeling` head on top.""", MRA_START_DOCSTRING) +@auto_docstring class MraForMaskedLM(MraPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -1043,12 +961,7 @@ class MraForMaskedLM(MraPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(MRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1122,10 +1035,11 @@ class MraClassificationHead(nn.Module): return x -@add_start_docstrings( - """MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of - the pooled output) e.g. for GLUE tasks.""", - MRA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. + """ ) class MraForSequenceClassification(MraPreTrainedModel): def __init__(self, config): @@ -1137,12 +1051,7 @@ class MraForSequenceClassification(MraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1211,11 +1120,7 @@ class MraForSequenceClassification(MraPreTrainedModel): ) -@add_start_docstrings( - """MRA Model with a multiple choice classification head on top (a linear layer on top of - the pooled output and a softmax) e.g. for RocStories/SWAG tasks.""", - MRA_START_DOCSTRING, -) +@auto_docstring class MraForMultipleChoice(MraPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1227,12 +1132,7 @@ class MraForMultipleChoice(MraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MRA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1246,6 +1146,30 @@ class MraForMultipleChoice(MraPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1300,11 +1224,7 @@ class MraForMultipleChoice(MraPreTrainedModel): ) -@add_start_docstrings( - """MRA Model with a token classification head on top (a linear layer on top of - the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.""", - MRA_START_DOCSTRING, -) +@auto_docstring class MraForTokenClassification(MraPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1317,12 +1237,7 @@ class MraForTokenClassification(MraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1383,11 +1298,7 @@ class MraForTokenClassification(MraPreTrainedModel): ) -@add_start_docstrings( - """MRA Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`).""", - MRA_START_DOCSTRING, -) +@auto_docstring class MraForQuestionAnswering(MraPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1401,12 +1312,7 @@ class MraForQuestionAnswering(MraPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1420,16 +1326,6 @@ class MraForQuestionAnswering(MraPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.mra( diff --git a/src/transformers/models/mt5/modeling_mt5.py b/src/transformers/models/mt5/modeling_mt5.py index 855e967c5f..500b2ca431 100644 --- a/src/transformers/models/mt5/modeling_mt5.py +++ b/src/transformers/models/mt5/modeling_mt5.py @@ -43,12 +43,11 @@ from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from ...utils.model_parallel_utils import assert_device_map, get_device_map from .configuration_mt5 import MT5Config @@ -61,9 +60,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MT5Config" -_CHECKPOINT_FOR_DOC = "mt5-small" - #################################################### # This dict contains ids and associated url @@ -752,13 +748,9 @@ class MT5ClassificationHead(nn.Module): return hidden_states +@auto_docstring # Copied from transformers.models.t5.modeling_t5.T5PreTrainedModel with T5->MT5, t5->mt5 class MT5PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MT5Config load_tf_weights = load_tf_weights_in_mt5 base_model_prefix = "transformer" @@ -1314,157 +1306,6 @@ class MT5Stack(MT5PreTrainedModel): return causal_mask -MT5_START_DOCSTRING = r""" - - The MT5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text - Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan - Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a - text-to-text denoising generative setting. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MT5Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MT5_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you - should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - [What are input IDs?](../glossary#input-ids) - - To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5 - Training](./mt5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -MT5_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you - should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask __HEAD_MASK_WARNING_MSG = """ The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently, @@ -1474,10 +1315,7 @@ num_heads)`. """ -@add_start_docstrings( - "The bare MT5 Model transformer outputting raw hidden-states without any specific head on top.", - MT5_START_DOCSTRING, -) +@auto_docstring class MT5Model(MT5PreTrainedModel): r""" Examples: @@ -1587,8 +1425,7 @@ class MT5Model(MT5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Copied from transformers.models.t5.modeling_t5.T5Model.forward with google-t5/->google/, T5->MT5, t5->mt5 def forward( self, @@ -1610,7 +1447,44 @@ class MT5Model(MT5PreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5 + Training](./mt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1705,7 +1579,11 @@ class MT5Model(MT5PreTrainedModel): ) -@add_start_docstrings("""MT5 Model with a `language modeling` head on top.""", MT5_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + MT5 Model with a `language modeling` head on top. + """ +) class MT5ForConditionalGeneration(MT5PreTrainedModel, GenerationMixin): r""" Examples: @@ -1819,8 +1697,7 @@ class MT5ForConditionalGeneration(MT5PreTrainedModel, GenerationMixin): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(MT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Copied from transformers.models.t5.modeling_t5.T5ForConditionalGeneration.forward with google-t5/->google/, T5->MT5, t5->mt5 def forward( self, @@ -1843,13 +1720,49 @@ class MT5ForConditionalGeneration(MT5PreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5 + Training](./mt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -2013,10 +1926,7 @@ class MT5ForConditionalGeneration(MT5PreTrainedModel, GenerationMixin): return reordered_decoder_past -@add_start_docstrings( - "The bare MT5 Model transformer outputting encoder's raw hidden-states without any specific head on top.", - MT5_START_DOCSTRING, -) +@auto_docstring class MT5EncoderModel(MT5PreTrainedModel): r""" Examples: @@ -2107,8 +2017,7 @@ class MT5EncoderModel(MT5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.block[layer].layer[0].SelfAttention.prune_heads(heads) - @add_start_docstrings_to_model_forward(MT5_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Copied from transformers.models.t5.modeling_t5.T5EncoderModel.forward with google-t5/->google/, T5->MT5, t5->mt5 def forward( self, @@ -2121,7 +2030,14 @@ class MT5EncoderModel(MT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). Example: @@ -2151,12 +2067,11 @@ class MT5EncoderModel(MT5PreTrainedModel): return encoder_outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" MT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MT5_START_DOCSTRING, + """ ) class MT5ForSequenceClassification(MT5PreTrainedModel): _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"] @@ -2173,9 +2088,8 @@ class MT5ForSequenceClassification(MT5PreTrainedModel): self.model_parallel = False - @add_start_docstrings_to_model_forward(MT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) - # Copied from transformers.models.t5.modeling_t5.T5ForSequenceClassification.forward + @auto_docstring + # Copied from transformers.models.t5.modeling_t5.T5ForSequenceClassification.forward with T5->MT5, t5->mt5 def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2195,10 +2109,47 @@ class MT5ForSequenceClassification(MT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5 + Training](./mt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -2209,7 +2160,7 @@ class MT5ForSequenceClassification(MT5PreTrainedModel): f"Passing input embeddings is currently not supported for {self.__class__.__name__}" ) - # Copied from models.bart.modeling_bart.BartModel.forward different to other models, T5 automatically creates + # Copied from models.bart.modeling_bart.BartModel.forward different to other models, MT5 automatically creates # decoder_input_ids from input_ids if no decoder_input_ids are provided if decoder_input_ids is None and decoder_inputs_embeds is None: if input_ids is None: @@ -2286,13 +2237,7 @@ class MT5ForSequenceClassification(MT5PreTrainedModel): ) -@add_start_docstrings( - """ - MT5 Encoder Model with a token classification head on top (a linear layer on top of the hidden-states output) - e.g. for Named-Entity-Recognition (NER) tasks. - """, - MT5_START_DOCSTRING, -) +@auto_docstring class MT5ForTokenClassification(MT5PreTrainedModel): _tied_weights_keys = ["transformer.encoder.embed_tokens.weight"] @@ -2308,8 +2253,7 @@ class MT5ForTokenClassification(MT5PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(MT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Copied from transformers.models.t5.modeling_t5.T5ForTokenClassification.forward with T5->MT5 def forward( self, @@ -2323,9 +2267,18 @@ class MT5ForTokenClassification(MT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./t5#training). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2360,13 +2313,7 @@ class MT5ForTokenClassification(MT5PreTrainedModel): ) -@add_start_docstrings( - """ - MT5 Model with a span classification head on top for extractive question-answering tasks like SQuAD (linear layers - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MT5_START_DOCSTRING, -) +@auto_docstring class MT5ForQuestionAnswering(MT5PreTrainedModel): _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"] _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -2416,8 +2363,7 @@ class MT5ForQuestionAnswering(MT5PreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(MT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Copied from transformers.models.t5.modeling_t5.T5ForQuestionAnswering.forward def forward( self, @@ -2439,15 +2385,44 @@ class MT5ForQuestionAnswering(MT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict use_cache = use_cache if use_cache is not None else self.config.use_cache diff --git a/src/transformers/models/musicgen/modeling_musicgen.py b/src/transformers/models/musicgen/modeling_musicgen.py index d38aeab3f8..f906c1a93c 100644 --- a/src/transformers/models/musicgen/modeling_musicgen.py +++ b/src/transformers/models/musicgen/modeling_musicgen.py @@ -49,12 +49,7 @@ from ...modeling_outputs import ( Seq2SeqLMOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel from .configuration_musicgen import MusicgenConfig, MusicgenDecoderConfig @@ -68,9 +63,6 @@ if TYPE_CHECKING: logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MusicgenConfig" -_CHECKPOINT_FOR_DOC = "facebook/musicgen-small" - @dataclass class MusicgenUnconditionalInput(ModelOutput): @@ -693,12 +685,8 @@ class MusicgenDecoderLayer(nn.Module): return outputs +@auto_docstring class MusicgenPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MusicgenDecoderConfig base_model_prefix = "model" supports_gradient_checkpointing = True @@ -718,206 +706,6 @@ class MusicgenPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -MUSICGEN_START_DOCSTRING = r""" - - The Musicgen model was proposed in [Simple and Controllable Music Generation](https://arxiv.org/abs/2306.05284) by - Jade Copet, Felix Kreuk, Itai Gat, Tal Remez, David Kant, Gabriel Synnaeve, Yossi Adi, Alexandre Défossez. It is an - encoder decoder transformer trained on the task of conditional music generation - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MusicgenConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MUSICGEN_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. - - Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, - such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - - - The `decoder_input_ids` will automatically be converted from shape `(batch_size * num_codebooks, - target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If - you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of - frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, - target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as - `decoder_input_ids`. - - - - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): - Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -MUSICGEN_DECODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): - Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. - - Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, - such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. - - [What are input IDs?](../glossary#input-ids) - - - - The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, - target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If - you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of - frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, - target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as - `input_ids`. - - - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of - the decoder. - encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing - cross-attention on hidden heads. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class MusicgenDecoder(MusicgenPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MusicgenDecoderLayer`] @@ -956,7 +744,7 @@ class MusicgenDecoder(MusicgenPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(MUSICGEN_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -972,6 +760,43 @@ class MusicgenDecoder(MusicgenPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of + the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing + cross-attention on hidden heads. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1132,10 +957,7 @@ class MusicgenDecoder(MusicgenPreTrainedModel): ) -@add_start_docstrings( - "The bare Musicgen decoder model outputting raw hidden-states without any specific head on top.", - MUSICGEN_START_DOCSTRING, -) +@auto_docstring class MusicgenModel(MusicgenPreTrainedModel): def __init__(self, config: MusicgenDecoderConfig): super().__init__(config) @@ -1152,7 +974,7 @@ class MusicgenModel(MusicgenPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(MUSICGEN_DECODER_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1168,6 +990,43 @@ class MusicgenModel(MusicgenPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of + the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing + cross-attention on hidden heads. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1203,9 +1062,10 @@ class MusicgenModel(MusicgenPreTrainedModel): ) -@add_start_docstrings( - "The MusicGen decoder model with a language modelling head on top.", - MUSICGEN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The MusicGen decoder model with a language modelling head on top. + """ ) class MusicgenForCausalLM(MusicgenPreTrainedModel, GenerationMixin): def __init__(self, config: MusicgenDecoderConfig): @@ -1239,8 +1099,7 @@ class MusicgenForCausalLM(MusicgenPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @add_start_docstrings_to_model_forward(MUSICGEN_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1259,11 +1118,45 @@ class MusicgenForCausalLM(MusicgenPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of + the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing + cross-attention on hidden heads. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1659,10 +1552,10 @@ class MusicgenForCausalLM(MusicgenPreTrainedModel, GenerationMixin): return output_ids -@add_start_docstrings( - "The composite MusicGen model with a text encoder, audio encoder and Musicgen decoder, " - "for music generation tasks with one or both of text and audio prompts.", - MUSICGEN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The composite MusicGen model with a text encoder, audio encoder and Musicgen decoder, + """ ) class MusicgenForConditionalGeneration(PreTrainedModel, GenerationMixin): config_class = MusicgenConfig @@ -1679,6 +1572,14 @@ class MusicgenForConditionalGeneration(PreTrainedModel, GenerationMixin): audio_encoder: Optional[PreTrainedModel] = None, decoder: Optional[MusicgenForCausalLM] = None, ): + r""" + text_encoder (`PreTrainedModel`, *optional*): + The text encoder model that encodes text into hidden states for conditioning. + audio_encoder (`PreTrainedModel`, *optional*): + The audio encoder model that encodes audio into hidden states for conditioning. + decoder (`MusicgenForCausalLM`, *optional*): + The decoder model that generates audio tokens based on conditioning signals. + """ if config is None and (text_encoder is None or audio_encoder is None or decoder is None): raise ValueError( "Either a configuration has to be provided, or all three of text encoder, audio encoder and MusicGen decoder." @@ -2000,8 +1901,7 @@ class MusicgenForConditionalGeneration(PreTrainedModel, GenerationMixin): ) return cls(text_encoder=text_encoder, audio_encoder=audio_encoder, decoder=decoder, config=config) - @add_start_docstrings_to_model_forward(MUSICGEN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2022,7 +1922,38 @@ class MusicgenForConditionalGeneration(PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + + + The `decoder_input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `decoder_input_ids`. + + + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + padding_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) Examples: ```python diff --git a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py index a35cdaa407..2b6930ff3a 100644 --- a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -36,17 +36,9 @@ from ...generation import ( ) from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available -from ...modeling_outputs import ( - BaseModelOutputWithPast, - ModelOutput, -) +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel, AutoModelForTextEncoding from .configuration_musicgen_melody import MusicgenMelodyConfig, MusicgenMelodyDecoderConfig @@ -60,9 +52,6 @@ if TYPE_CHECKING: logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "MusicgenMelodyConfig" -_CHECKPOINT_FOR_DOC = "facebook/musicgen-melody" - @dataclass class MusicgenMelodyOutputWithPast(ModelOutput): @@ -651,13 +640,9 @@ class MusicgenMelodyDecoderLayer(nn.Module): return outputs +@auto_docstring # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenPreTrainedModel with Musicgen->MusicgenMelody class MusicgenMelodyPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = MusicgenMelodyDecoderConfig base_model_prefix = "model" supports_gradient_checkpointing = True @@ -677,178 +662,6 @@ class MusicgenMelodyPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -MUSICGEN_MELODY_START_DOCSTRING = r""" - - The Musicgen Melody model was proposed in [Simple and Controllable Music Generation](https://arxiv.org/abs/2306.05284) by - Jade Copet, Felix Kreuk, Itai Gat, Tal Remez, David Kant, Gabriel Synnaeve, Yossi Adi, Alexandre Défossez. It is a - decoder-only transformer trained on the task of conditional music generation. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MusicgenMelodyConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MUSICGEN_MELODY_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - input_features (`torch.FloatTensor` of shape `(batch_size, audio_sequence_length, num_chroma)`): - Input audio features. - This should be returned by the [`MusicgenMelodyFeatureExtractor`] class that you can also - retrieve from [`AutoFeatureExtractor`]. See [`MusicgenMelodyFeatureExtractor.__call__`] for details. - decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. - - Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, - such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - - - The `decoder_input_ids` will automatically be converted from shape `(batch_size * num_codebooks, - target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If - you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of - frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, - target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as - `decoder_input_ids`. - - - - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, encoder_sequence_length + sequence_length, embed_size_per_head)`). - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of conditional hidden-states representing the concatenation of the projected text encoder output and the projected audio encoder output. - Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): - Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set - `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` - are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -MUSICGEN_MELODY_DECODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): - Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. - - Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, - such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. - - [What are input IDs?](../glossary#input-ids) - - - - The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, - target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If - you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of - frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, - target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as - `input_ids`. - - - - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. - Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. - encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing attention on conditional hidden states. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenDecoder with MUSICGEN->MUSICGEN_MELODY,Musicgen->MusicgenMelody class MusicgenMelodyDecoder(MusicgenMelodyPreTrainedModel): """ @@ -888,7 +701,7 @@ class MusicgenMelodyDecoder(MusicgenMelodyPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(MUSICGEN_MELODY_DECODER_INPUTS_DOCSTRING) + @auto_docstring # Ignore copy def forward( self, @@ -904,6 +717,37 @@ class MusicgenMelodyDecoder(MusicgenMelodyPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing attention on conditional hidden states. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1043,10 +887,7 @@ class MusicgenMelodyDecoder(MusicgenMelodyPreTrainedModel): ) -@add_start_docstrings( - "The bare MusicgenMelody decoder model outputting raw hidden-states without any specific head on top.", - MUSICGEN_MELODY_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenModel with MUSICGEN->MUSICGEN_MELODY,Musicgen->MusicgenMelody class MusicgenMelodyModel(MusicgenMelodyPreTrainedModel): def __init__(self, config: MusicgenMelodyDecoderConfig): @@ -1064,7 +905,7 @@ class MusicgenMelodyModel(MusicgenMelodyPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(MUSICGEN_MELODY_DECODER_INPUTS_DOCSTRING) + @auto_docstring # Ignore copy def forward( self, @@ -1080,6 +921,37 @@ class MusicgenMelodyModel(MusicgenMelodyPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing attention on conditional hidden states. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1113,9 +985,10 @@ class MusicgenMelodyModel(MusicgenMelodyPreTrainedModel): ) -@add_start_docstrings( - "The Musicgen Melody decoder model with a language modelling head on top.", - MUSICGEN_MELODY_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Musicgen Melody decoder model with a language modelling head on top. + """ ) # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenForCausalLM with MUSICGEN->MUSICGEN_MELODY,Musicgen->MusicgenMelody,MusicGen->Musicgen Melody class MusicgenMelodyForCausalLM(MusicgenMelodyPreTrainedModel, GenerationMixin): @@ -1150,8 +1023,7 @@ class MusicgenMelodyForCausalLM(MusicgenMelodyPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @add_start_docstrings_to_model_forward(MUSICGEN_MELODY_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MusicgenMelodyOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Ignore copy def forward( self, @@ -1169,11 +1041,39 @@ class MusicgenMelodyForCausalLM(MusicgenMelodyPreTrainedModel, GenerationMixin): labels: Optional[torch.LongTensor] = None, ) -> Union[Tuple, MusicgenMelodyOutputWithPast]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing attention on conditional hidden states. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1580,16 +1480,7 @@ class MusicgenMelodyForCausalLM(MusicgenMelodyPreTrainedModel, GenerationMixin): return output_ids -@add_start_docstrings( - "The composite Musicgen Melody model with a text and audio conditional models, a MusicgenMelody decoder and an audio encoder, " - "for music generation tasks with one or both of text and audio prompts.", - MUSICGEN_MELODY_START_DOCSTRING, - """ - text_encoder (`Optional[PreTrainedModel]`, *optional*): Text encoder. - audio_encoder (`Optional[PreTrainedModel]`, *optional*): Audio code decoder. - decoder (`Optional[MusicgenMelodyForCausalLM]`, *optional*): MusicGen Melody decoder used to generate audio codes. - """, -) +@auto_docstring class MusicgenMelodyForConditionalGeneration(PreTrainedModel, GenerationMixin): config_class = MusicgenMelodyConfig main_input_name = "input_ids" @@ -1604,6 +1495,14 @@ class MusicgenMelodyForConditionalGeneration(PreTrainedModel, GenerationMixin): audio_encoder: Optional[PreTrainedModel] = None, decoder: Optional[MusicgenMelodyForCausalLM] = None, ): + r""" + text_encoder (`PreTrainedModel`, *optional*): + The text encoder model that encodes text into hidden states for conditioning. + audio_encoder (`PreTrainedModel`, *optional*): + The audio encoder model that encodes audio into hidden states for conditioning. + decoder (`MusicgenForCausalLM`, *optional*): + The decoder model that generates audio tokens based on conditioning signals. + """ if config is None and None in (text_encoder, audio_encoder, decoder): raise ValueError( "Either a configuration has to be provided, or all three of text encoder, audio encoder and Musicgen Melody decoder." @@ -1901,8 +1800,7 @@ class MusicgenMelodyForConditionalGeneration(PreTrainedModel, GenerationMixin): ) return cls(text_encoder=text_encoder, audio_encoder=audio_encoder, decoder=decoder, config=config) - @add_start_docstrings_to_model_forward(MUSICGEN_MELODY_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MusicgenMelodyOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1922,7 +1820,38 @@ class MusicgenMelodyForConditionalGeneration(PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, MusicgenMelodyOutputWithPast]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, audio_sequence_length, num_chroma)`): + Input audio features. + This should be returned by the [`MusicgenMelodyFeatureExtractor`] class that you can also + retrieve from [`AutoFeatureExtractor`]. See [`MusicgenMelodyFeatureExtractor.__call__`] for details. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + + + The `decoder_input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `decoder_input_ids`. + + + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of conditional hidden-states representing the concatenation of the projected text encoder output and the projected audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` Examples: ```python diff --git a/src/transformers/models/mvp/modeling_mvp.py b/src/transformers/models/mvp/modeling_mvp.py index 2dec79729c..1c38623496 100644 --- a/src/transformers/models/mvp/modeling_mvp.py +++ b/src/transformers/models/mvp/modeling_mvp.py @@ -36,25 +36,12 @@ from ...modeling_outputs import ( Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_mvp import MvpConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "RUCAIBox/mvp" -_CONFIG_FOR_DOC = "MvpConfig" - -# Base model docstring -_EXPECTED_OUTPUT_SHAPE = [1, 8, 1024] - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -498,6 +485,7 @@ class MvpPrompt(nn.Module): return prompt +@auto_docstring class MvpPreTrainedModel(PreTrainedModel): config_class = MvpConfig base_model_prefix = "model" @@ -525,214 +513,6 @@ class MvpPreTrainedModel(PreTrainedModel): return dummy_inputs -MVP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`MvpConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -MVP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -MVP_CONDITIONAL_GENERATION_EXAMPLE = r""" - Example of summarization: - - Fine-tuning a model - ```python - >>> import torch - >>> from transformers import AutoTokenizer, MvpForConditionalGeneration - - >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") - >>> model = MvpForConditionalGeneration.from_pretrained("RUCAIBox/mvp") - - >>> inputs = tokenizer( - ... "Summarize: You may want to stick it to your boss and leave your job, but don't do it if these are your reasons.", - ... return_tensors="pt", - ... ) - >>> labels = tokenizer("Bad Reasons To Quit Your Job", return_tensors="pt")["input_ids"] - - >>> loss = model(**inputs, labels=labels).loss - >>> loss.backward() - ``` - - Inference after the model fine-tuned - ```python - >>> with torch.no_grad(): - ... generated_ids = model.generate(**inputs) - - >>> generated_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True) - ``` -""" - -MVP_SEQUENCE_CLASSIFICATION_SAMPLE = r""" - Example of single-label classification: - - Fine-tuning a model on `num_labels` classes - ```python - >>> import torch - >>> from transformers import AutoTokenizer, MvpForSequenceClassification - - >>> num_labels = 2 # for example, this is a binary classification task - >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") - >>> model = MvpForSequenceClassification.from_pretrained("RUCAIBox/mvp", num_labels=num_labels) - - >>> inputs = tokenizer("Classify: Hello, my dog is cute", return_tensors="pt") - >>> labels = torch.tensor(1) # the real label for inputs - - >>> loss = model(**inputs, labels=labels).loss - >>> loss.backward() - ``` - - Inference after the model fine-tuned - ```python - >>> with torch.no_grad(): - ... logits = model(**inputs).logits - - >>> predicted_class_id = logits.argmax() - ``` -""" - -MVP_QUESTION_ANSWERING_SAMPLE = r""" - Example: - - Fine-tuning a model for extrative question answering, and our model also supports generative question answering - using `BartForConditionalGeneration` - ```python - >>> import torch - >>> from transformers import AutoTokenizer, MvpForQuestionAnswering - - >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") - >>> model = MvpForQuestionAnswering.from_pretrained("RUCAIBox/mvp") - - >>> inputs = tokenizer( - ... "Answer the following question: Who was Jim Henson? [SEP] Jim Henson was a nice puppet", - ... return_tensors="pt", - ... ) - >>> target_start_index = torch.tensor([18]) - >>> target_end_index = torch.tensor([19]) - - >>> loss = model(**inputs, start_positions=target_start_index, end_positions=target_end_index).loss - >>> loss.backward() - ``` - - Inference after the model fine-tuned - ```python - >>> with torch.no_grad(): - ... outputs = model(**inputs) - - >>> answer_start_index = outputs.start_logits.argmax() - >>> answer_end_index = outputs.end_logits.argmax() - - >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1] - >>> predict_answer = tokenizer.decode(predict_answer_tokens) - ``` -""" - - class MvpEncoder(MvpPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1211,10 +991,7 @@ class MvpDecoder(MvpPreTrainedModel): ) -@add_start_docstrings( - "The bare MVP Model outputting raw hidden-states without any specific head on top.", - MVP_START_DOCSTRING, -) +@auto_docstring class MvpModel(MvpPreTrainedModel): _keys_to_ignore_on_load_unexpected = ["final_logits_bias"] _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1254,13 +1031,7 @@ class MvpModel(MvpPreTrainedModel): self.decoder.self_attn_prompt.requires_grad_(True) self.decoder.cross_attn_prompt.requires_grad_(True) - @add_start_docstrings_to_model_forward(MVP_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1279,6 +1050,35 @@ class MvpModel(MvpPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ # different to other models, Mvp automatically creates decoder_input_ids from # input_ids if no decoder_input_ids are provided if decoder_input_ids is None and decoder_inputs_embeds is None: @@ -1349,8 +1149,10 @@ class MvpModel(MvpPreTrainedModel): ) -@add_start_docstrings( - "The MVP Model with a language modeling head. Can be used for various text generation tasks.", MVP_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The MVP Model with a language modeling head. Can be used for various text generation tasks. + """ ) class MvpForConditionalGeneration(MvpPreTrainedModel, GenerationMixin): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"] @@ -1396,9 +1198,7 @@ class MvpForConditionalGeneration(MvpPreTrainedModel, GenerationMixin): self.model.set_lightweight_tuning() self.lm_head.requires_grad_(False) - @add_start_docstrings_to_model_forward(MVP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(MVP_CONDITIONAL_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1419,12 +1219,65 @@ class MvpForConditionalGeneration(MvpPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example of summarization: + + Fine-tuning a model + ```python + >>> import torch + >>> from transformers import AutoTokenizer, MvpForConditionalGeneration + + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") + >>> model = MvpForConditionalGeneration.from_pretrained("RUCAIBox/mvp") + + >>> inputs = tokenizer( + ... "Summarize: You may want to stick it to your boss and leave your job, but don't do it if these are your reasons.", + ... return_tensors="pt", + ... ) + >>> labels = tokenizer("Bad Reasons To Quit Your Job", return_tensors="pt")["input_ids"] + + >>> loss = model(**inputs, labels=labels).loss + >>> loss.backward() + ``` + + Inference after the model fine-tuned + ```python + >>> with torch.no_grad(): + ... generated_ids = model.generate(**inputs) + + >>> generated_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True) + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1492,12 +1345,11 @@ class MvpForConditionalGeneration(MvpPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Mvp model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - MVP_START_DOCSTRING, + """ ) class MvpForSequenceClassification(MvpPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1519,8 +1371,7 @@ class MvpForSequenceClassification(MvpPreTrainedModel): self.model.set_lightweight_tuning() self.classification_head.requires_grad_(False) - @add_start_docstrings_to_model_forward(MVP_INPUTS_DOCSTRING) - @add_end_docstrings(MVP_SEQUENCE_CLASSIFICATION_SAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1540,9 +1391,62 @@ class MvpForSequenceClassification(MvpPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Example of single-label classification: + + Fine-tuning a model on `num_labels` classes + ```python + >>> import torch + >>> from transformers import AutoTokenizer, MvpForSequenceClassification + + >>> num_labels = 2 # for example, this is a binary classification task + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") + >>> model = MvpForSequenceClassification.from_pretrained("RUCAIBox/mvp", num_labels=num_labels) + + >>> inputs = tokenizer("Classify: Hello, my dog is cute", return_tensors="pt") + >>> labels = torch.tensor(1) # the real label for inputs + + >>> loss = model(**inputs, labels=labels).loss + >>> loss.backward() + ``` + + Inference after the model fine-tuned + ```python + >>> with torch.no_grad(): + ... logits = model(**inputs).logits + + >>> predicted_class_id = logits.argmax() + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -1619,13 +1523,7 @@ class MvpForSequenceClassification(MvpPreTrainedModel): ) -@add_start_docstrings( - """ - MVP Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layer - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - MVP_START_DOCSTRING, -) +@auto_docstring class MvpForQuestionAnswering(MvpPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1645,8 +1543,7 @@ class MvpForQuestionAnswering(MvpPreTrainedModel): self.model.set_lightweight_tuning() self.qa_outputs.requires_grad_(False) - @add_start_docstrings_to_model_forward(MVP_INPUTS_DOCSTRING) - @add_end_docstrings(MVP_QUESTION_ANSWERING_SAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1667,14 +1564,67 @@ class MvpForQuestionAnswering(MvpPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Mvp uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_mvp._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + Example: + + Fine-tuning a model for extrative question answering, and our model also supports generative question answering + using `BartForConditionalGeneration` + ```python + >>> import torch + >>> from transformers import AutoTokenizer, MvpForQuestionAnswering + + >>> tokenizer = AutoTokenizer.from_pretrained("RUCAIBox/mvp") + >>> model = MvpForQuestionAnswering.from_pretrained("RUCAIBox/mvp") + + >>> inputs = tokenizer( + ... "Answer the following question: Who was Jim Henson? [SEP] Jim Henson was a nice puppet", + ... return_tensors="pt", + ... ) + >>> target_start_index = torch.tensor([18]) + >>> target_end_index = torch.tensor([19]) + + >>> loss = model(**inputs, start_positions=target_start_index, end_positions=target_end_index).loss + >>> loss.backward() + ``` + + Inference after the model fine-tuned + ```python + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> answer_start_index = outputs.start_logits.argmax() + >>> answer_end_index = outputs.end_logits.argmax() + + >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1] + >>> predict_answer = tokenizer.decode(predict_answer_tokens) + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if start_positions is not None and end_positions is not None: @@ -1794,7 +1744,7 @@ class MvpForCausalLM(MvpPreTrainedModel, GenerationMixin): self.model.set_lightweight_tuning() self.lm_head.requires_grad_(False) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1812,72 +1762,15 @@ class MvpForCausalLM(MvpPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/nemotron/modeling_nemotron.py b/src/transformers/models/nemotron/modeling_nemotron.py index aaca0943c9..3e1ce01041 100644 --- a/src/transformers/models/nemotron/modeling_nemotron.py +++ b/src/transformers/models/nemotron/modeling_nemotron.py @@ -38,15 +38,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_nemotron import NemotronConfig @@ -58,9 +50,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "nvidia/nemotron-3-8b-base-4k-hf" -_CONFIG_FOR_DOC = "NemotronConfig" - def _cast_if_autocast_enabled(*args): if not torch.is_autocast_enabled(): @@ -579,27 +568,7 @@ class NemotronDecoderLayer(nn.Module): return outputs -NEMOTRON_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`NemotronConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Nemotron Model outputting raw hidden-states without any specific head on top.", - NEMOTRON_START_DOCSTRING, -) +@auto_docstring class NemotronPreTrainedModel(PreTrainedModel): config_class = NemotronConfig base_model_prefix = "model" @@ -627,85 +596,7 @@ class NemotronPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -NEMOTRON_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Nemotron Model outputting raw hidden-states without any specific head on top.", - NEMOTRON_START_DOCSTRING, -) +@auto_docstring class NemotronModel(NemotronPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`NemotronDecoderLayer`] @@ -737,7 +628,7 @@ class NemotronModel(NemotronPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(NEMOTRON_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1006,9 +897,7 @@ class NemotronForCausalLM(NemotronPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(NEMOTRON_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - # Ignore copy (doc string different) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1025,19 +914,10 @@ class NemotronForCausalLM(NemotronPreTrainedModel, GenerationMixin): **loss_kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1091,8 +971,8 @@ class NemotronForCausalLM(NemotronPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Nemotron Model transformer with a sequence classification head on top (linear layer). [`NemotronForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1103,8 +983,7 @@ class NemotronForCausalLM(NemotronPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - NEMOTRON_START_DOCSTRING, + """ ) # Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with LLAMA->NEMOTRON,Llama->Nemotron,llama->nemotron class NemotronForSequenceClassification(NemotronPreTrainedModel): @@ -1124,7 +1003,7 @@ class NemotronForSequenceClassification(NemotronPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(NEMOTRON_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1193,13 +1072,7 @@ class NemotronForSequenceClassification(NemotronPreTrainedModel): ) -@add_start_docstrings( - """ -The Nemotron Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - NEMOTRON_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.llama.modeling_llama.LlamaForQuestionAnswering with LLAMA->NEMOTRON,Llama->Nemotron,llama->nemotron class NemotronForQuestionAnswering(NemotronPreTrainedModel): base_model_prefix = "transformer" @@ -1220,7 +1093,7 @@ class NemotronForQuestionAnswering(NemotronPreTrainedModel): self.transformer.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(NEMOTRON_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1234,17 +1107,6 @@ class NemotronForQuestionAnswering(NemotronPreTrainedModel): output_hidden_states: Optional[bool] = None, **kwargs, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.transformer( input_ids, attention_mask=attention_mask, @@ -1275,13 +1137,7 @@ class NemotronForQuestionAnswering(NemotronPreTrainedModel): ) -@add_start_docstrings( - """ - The Nemotron Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - NEMOTRON_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.llama.modeling_llama.LlamaForTokenClassification with LLAMA->NEMOTRON,Llama->Nemotron,llama->nemotron class NemotronForTokenClassification(NemotronPreTrainedModel): def __init__(self, config): @@ -1307,12 +1163,7 @@ class NemotronForTokenClassification(NemotronPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(NEMOTRON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/nllb_moe/modeling_nllb_moe.py b/src/transformers/models/nllb_moe/modeling_nllb_moe.py index e8b80221c4..79002aee31 100644 --- a/src/transformers/models/nllb_moe/modeling_nllb_moe.py +++ b/src/transformers/models/nllb_moe/modeling_nllb_moe.py @@ -33,22 +33,12 @@ from ...modeling_outputs import ( Seq2SeqMoEOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_nllb_moe import NllbMoeConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "NllbMoeConfig" -_CHECKPOINT_FOR_DOC = "hf-internal-testing/dummy-nllb-moe-2-experts" -_REAL_CHECKPOINT_FOR_DOC = "facebook/nllb-moe-54b" - #################################################### # This dict contains ids and associated url @@ -841,6 +831,7 @@ class NllbMoeDecoderLayer(nn.Module): return outputs +@auto_docstring class NllbMoePreTrainedModel(PreTrainedModel): config_class = NllbMoeConfig base_model_prefix = "model" @@ -860,133 +851,6 @@ class NllbMoePreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -NLLB_MOE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`NllbMoeConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -NLLB_MOE_GENERATION_EXAMPLE = r""" - Translation example: - - ```python - >>> from transformers import AutoTokenizer, NllbMoeForConditionalGeneration - - >>> model = NllbMoeForConditionalGeneration.from_pretrained("facebook/nllb-moe-54b") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-moe-54b") - - >>> text_to_translate = "Life is like a box of chocolates" - >>> model_inputs = tokenizer(text_to_translate, return_tensors="pt") - - >>> # translate to French - >>> gen_tokens = model.generate(**model_inputs, forced_bos_token_id=tokenizer.get_lang_id("eng_Latn")) - >>> print(tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)) - ``` -""" - -NLLB_MOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - NllbMoe uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class NllbMoeEncoder(NllbMoePreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1467,10 +1331,7 @@ class NllbMoeDecoder(NllbMoePreTrainedModel): ) -@add_start_docstrings( - "The bare NllbMoe Model outputting raw hidden-states without any specific head on top.", - NLLB_MOE_START_DOCSTRING, -) +@auto_docstring class NllbMoeModel(NllbMoePreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1506,9 +1367,7 @@ class NllbMoeModel(NllbMoePreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING) - @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqMoEModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1529,7 +1388,26 @@ class NllbMoeModel(NllbMoePreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqMoEModelOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + NllbMoe uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1606,8 +1484,10 @@ class NllbMoeModel(NllbMoePreTrainedModel): ) -@add_start_docstrings( - "The NllbMoe Model with a language modeling head. Can be used for summarization.", NLLB_MOE_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The NllbMoe Model with a language modeling head. Can be used for summarization. + """ ) class NllbMoeForConditionalGeneration(NllbMoePreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1635,9 +1515,7 @@ class NllbMoeForConditionalGeneration(NllbMoePreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqMoEOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(NLLB_MOE_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1659,12 +1537,46 @@ class NllbMoeForConditionalGeneration(NllbMoePreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqMoEOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + NllbMoe uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Translation: + + ```python + >>> from transformers import AutoTokenizer, NllbMoeForConditionalGeneration + + >>> model = NllbMoeForConditionalGeneration.from_pretrained("facebook/nllb-moe-54b") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-moe-54b") + + >>> text_to_translate = "Life is like a box of chocolates" + >>> model_inputs = tokenizer(text_to_translate, return_tensors="pt") + + >>> # translate to French + >>> gen_tokens = model.generate(**model_inputs, forced_bos_token_id=tokenizer.get_lang_id("eng_Latn")) + >>> print(tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)) + ``` """ return_dict = return_dict if return_dict is not None else self.config.return_dict output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions diff --git a/src/transformers/models/nystromformer/modeling_nystromformer.py b/src/transformers/models/nystromformer/modeling_nystromformer.py index afa17d94e5..1f40b40c5d 100755 --- a/src/transformers/models/nystromformer/modeling_nystromformer.py +++ b/src/transformers/models/nystromformer/modeling_nystromformer.py @@ -33,15 +33,15 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import ( + auto_docstring, + logging, +) from .configuration_nystromformer import NystromformerConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "uw-madison/nystromformer-512" -_CONFIG_FOR_DOC = "NystromformerConfig" - class NystromformerEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings.""" @@ -442,12 +442,8 @@ class NystromformerOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class NystromformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = NystromformerConfig base_model_prefix = "nystromformer" supports_gradient_checkpointing = True @@ -469,71 +465,7 @@ class NystromformerPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -NYSTROMFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`NystromformerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -NYSTROMFORMER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Nyströmformer Model transformer outputting raw hidden-states without any specific head on top.", - NYSTROMFORMER_START_DOCSTRING, -) +@auto_docstring class NystromformerModel(NystromformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -559,12 +491,7 @@ class NystromformerModel(NystromformerPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -645,7 +572,7 @@ class NystromformerModel(NystromformerPreTrainedModel): ) -@add_start_docstrings("""Nyströmformer Model with a `language modeling` head on top.""", NYSTROMFORMER_START_DOCSTRING) +@auto_docstring class NystromformerForMaskedLM(NystromformerPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder"] @@ -665,12 +592,7 @@ class NystromformerForMaskedLM(NystromformerPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -745,12 +667,11 @@ class NystromformerClassificationHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Nyströmformer Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - NYSTROMFORMER_START_DOCSTRING, + """ ) class NystromformerForSequenceClassification(NystromformerPreTrainedModel): def __init__(self, config): @@ -762,12 +683,7 @@ class NystromformerForSequenceClassification(NystromformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -838,13 +754,7 @@ class NystromformerForSequenceClassification(NystromformerPreTrainedModel): ) -@add_start_docstrings( - """ - Nyströmformer Model with a multiple choice classification head on top (a linear layer on top of the pooled output - and a softmax) e.g. for RocStories/SWAG tasks. - """, - NYSTROMFORMER_START_DOCSTRING, -) +@auto_docstring class NystromformerForMultipleChoice(NystromformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -856,14 +766,7 @@ class NystromformerForMultipleChoice(NystromformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -878,6 +781,30 @@ class NystromformerForMultipleChoice(NystromformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -933,13 +860,7 @@ class NystromformerForMultipleChoice(NystromformerPreTrainedModel): ) -@add_start_docstrings( - """ - Nyströmformer Model with a token classification head on top (a linear layer on top of the hidden-states output) - e.g. for Named-Entity-Recognition (NER) tasks. - """, - NYSTROMFORMER_START_DOCSTRING, -) +@auto_docstring class NystromformerForTokenClassification(NystromformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -952,12 +873,7 @@ class NystromformerForTokenClassification(NystromformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1011,13 +927,7 @@ class NystromformerForTokenClassification(NystromformerPreTrainedModel): ) -@add_start_docstrings( - """ - Nyströmformer Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - NYSTROMFORMER_START_DOCSTRING, -) +@auto_docstring class NystromformerForQuestionAnswering(NystromformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1031,12 +941,7 @@ class NystromformerForQuestionAnswering(NystromformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(NYSTROMFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1051,16 +956,6 @@ class NystromformerForQuestionAnswering(NystromformerPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.nystromformer( diff --git a/src/transformers/models/olmo/modeling_olmo.py b/src/transformers/models/olmo/modeling_olmo.py index 2284f949f0..410dd25a96 100644 --- a/src/transformers/models/olmo/modeling_olmo.py +++ b/src/transformers/models/olmo/modeling_olmo.py @@ -20,15 +20,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_olmo import OlmoConfig @@ -39,7 +31,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "OlmoConfig" class OlmoLayerNorm(nn.Module): @@ -315,27 +306,7 @@ class OlmoRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -OLMO_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OlmoConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Olmo Model outputting raw hidden-states without any specific head on top.", - OLMO_START_DOCSTRING, -) +@auto_docstring class OlmoPreTrainedModel(PreTrainedModel): config_class = OlmoConfig base_model_prefix = "model" @@ -362,88 +333,8 @@ class OlmoPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -OLMO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Olmo Model outputting raw hidden-states without any specific head on top.", - OLMO_START_DOCSTRING, -) +@auto_docstring class OlmoModel(OlmoPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`OlmoDecoderLayer`] - - Args: - config: OlmoConfig - """ - def __init__(self, config: OlmoConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -467,7 +358,7 @@ class OlmoModel(OlmoPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(OLMO_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -689,6 +580,7 @@ class OlmoModel(OlmoPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class OlmoForCausalLM(OlmoPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -722,8 +614,7 @@ class OlmoForCausalLM(OlmoPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(OLMO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -740,19 +631,10 @@ class OlmoForCausalLM(OlmoPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/olmo2/modeling_olmo2.py b/src/transformers/models/olmo2/modeling_olmo2.py index 978c80dc5d..f87befb459 100644 --- a/src/transformers/models/olmo2/modeling_olmo2.py +++ b/src/transformers/models/olmo2/modeling_olmo2.py @@ -20,15 +20,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_olmo2 import Olmo2Config @@ -39,7 +31,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Olmo2Config" @use_kernel_forward_from_hub("RMSNorm") @@ -319,27 +310,7 @@ class Olmo2RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -OLMO2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Olmo2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Olmo2 Model outputting raw hidden-states without any specific head on top.", - OLMO2_START_DOCSTRING, -) +@auto_docstring class Olmo2PreTrainedModel(PreTrainedModel): config_class = Olmo2Config base_model_prefix = "model" @@ -368,88 +339,8 @@ class Olmo2PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -OLMO2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Olmo2 Model outputting raw hidden-states without any specific head on top.", - OLMO2_START_DOCSTRING, -) +@auto_docstring class Olmo2Model(Olmo2PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Olmo2DecoderLayer`] - - Args: - config: Olmo2Config - """ - def __init__(self, config: Olmo2Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -473,7 +364,7 @@ class Olmo2Model(Olmo2PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(OLMO2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -695,6 +586,7 @@ class Olmo2Model(Olmo2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Olmo2ForCausalLM(Olmo2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -728,8 +620,7 @@ class Olmo2ForCausalLM(Olmo2PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(OLMO2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -746,19 +637,10 @@ class Olmo2ForCausalLM(Olmo2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/olmoe/modeling_olmoe.py b/src/transformers/models/olmoe/modeling_olmoe.py index 429ea28413..88f884dc2e 100644 --- a/src/transformers/models/olmoe/modeling_olmoe.py +++ b/src/transformers/models/olmoe/modeling_olmoe.py @@ -24,19 +24,11 @@ from ...cache_utils import Cache, DynamicCache, StaticCache from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available -from ...modeling_outputs import ( - MoeCausalLMOutputWithPast, - MoeModelOutputWithPast, -) +from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_olmoe import OlmoeConfig @@ -46,8 +38,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "OlmoeConfig" - # Copied from transformers.models.mixtral.modeling_mixtral.load_balancing_loss_func def load_balancing_loss_func( @@ -707,27 +697,7 @@ class OlmoeDecoderLayer(nn.Module): return outputs -OLMOE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OlmoeConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Olmoe Model outputting raw hidden-states without any specific head on top.", - OLMOE_START_DOCSTRING, -) +@auto_docstring class OlmoePreTrainedModel(PreTrainedModel): config_class = OlmoeConfig base_model_prefix = "model" @@ -754,96 +724,8 @@ class OlmoePreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -OLMOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Olmoe Model outputting raw hidden-states without any specific head on top.", - OLMOE_START_DOCSTRING, -) -# TODO: re-enable check: Copied from transformers.models.llama.modeling_llama.LlamaModel with Llama->Olmoe +@auto_docstring class OlmoeModel(OlmoePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`OlmoeDecoderLayer`] - - Args: - config: OlmoeConfig - """ - def __init__(self, config: OlmoeConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -866,8 +748,7 @@ class OlmoeModel(OlmoePreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(OLMOE_INPUTS_DOCSTRING) - # Ignore copy + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1158,8 +1039,7 @@ class OlmoeForCausalLM(OlmoePreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(OLMOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1178,19 +1058,10 @@ class OlmoeForCausalLM(OlmoePreTrainedModel, GenerationMixin): **loss_kwargs, ) -> Union[Tuple, MoeCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py b/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py index 67143cff68..a5ae345e5d 100644 --- a/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py +++ b/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py @@ -28,21 +28,17 @@ from torch import Tensor, nn from ...activations import ACT2CLS, ACT2FN from ...file_utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, ) from ...integrations import use_kernel_forward_from_hub from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_utils import PreTrainedModel -from ...utils import logging +from ...utils import auto_docstring, logging from ...utils.backbone_utils import load_backbone from ..auto import AutoModel from .configuration_omdet_turbo import OmDetTurboConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "OmDetTurboConfig" @dataclass @@ -992,6 +988,7 @@ class OmDetTurboDeformableTransformerDecoderLayer(nn.Module): ) +@auto_docstring class OmDetTurboPreTrainedModel(PreTrainedModel): config_class = OmDetTurboConfig base_model_prefix = "model" @@ -1130,71 +1127,6 @@ class OmDetTurboPreTrainedModel(PreTrainedModel): return class_embeddings, task_embeddings, task_mask -OMDET_TURBO_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OmDetTurboConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -OMDET_TURBO_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`AutoImageProcessor`]. See [`DetrImageProcessor.__call__`] for - details. - - classes_input_ids (`torch.LongTensor` of shape `(total_classes (>= batch_size), sequence_length)`): - Indices of input classes sequence tokens in the vocabulary of the language model. - Several classes can be provided for each tasks, thus the tokenized classes are flattened - and the structure of the classes is provided in the `classes_structure` argument. - - Indices can be obtained using [`OmDetTurboProcessor`]. See [`OmDetTurboProcessor.__call__`] for - details. - - [What are input IDs?](../glossary#input-ids) - - classes_attention_mask (`torch.BoolTensor` of shape `(total_classes (>= batch_size), num_classes, sequence_length)`): - Attention mask for the classes. This is a binary mask that indicates which tokens should be attended to, - and which should not. - - tasks_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input tasks sequence tokens in the vocabulary of the language model. - - Indices can be obtained using [`OmDetTurboProcessor`]. See [`OmDetTurboProcessor.__call__`] for - details. - - [What are input IDs?](../glossary#input-ids) - - tasks_attention_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): - Attention mask for the tasks. This is a binary mask that indicates which tokens should be attended to, - and which should not. - - classes_structure (torch.LongTensor of shape `(batch_size)`): - Structure of the classes. This tensor indicates the number of classes for each task. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - """ - - def _cosine_similarity_scaled(a, b, logit_scale): a = a / a.norm(dim=2, keepdim=True).clamp_min(1e-12) b = b / b.norm(dim=1, keepdim=True).clamp_min(1e-12) @@ -1542,12 +1474,11 @@ class OmDetTurboDecoder(OmDetTurboPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" OmDetTurbo Model (consisting of a vision and a text backbone, and encoder-decoder architecture) outputting bounding boxes and classes scores for tasks such as COCO detection. - """, - OMDET_TURBO_START_DOCSTRING, + """ ) class OmDetTurboForObjectDetection(OmDetTurboPreTrainedModel): def __init__(self, config: OmDetTurboConfig): @@ -1579,8 +1510,7 @@ class OmDetTurboForObjectDetection(OmDetTurboPreTrainedModel): self.vocab_size = model_embeds.num_embeddings return model_embeds - @add_start_docstrings_to_model_forward(OMDET_TURBO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OmDetTurboObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1595,7 +1525,30 @@ class OmDetTurboForObjectDetection(OmDetTurboPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], OmDetTurboObjectDetectionOutput]: r""" - Returns: + classes_input_ids (`torch.LongTensor` of shape `(total_classes (>= batch_size), sequence_length)`): + Indices of input classes sequence tokens in the vocabulary of the language model. + Several classes can be provided for each tasks, thus the tokenized classes are flattened + and the structure of the classes is provided in the `classes_structure` argument. + + Indices can be obtained using [`OmDetTurboProcessor`]. See [`OmDetTurboProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + classes_attention_mask (`torch.BoolTensor` of shape `(total_classes (>= batch_size), num_classes, sequence_length)`): + Attention mask for the classes. This is a binary mask that indicates which tokens should be attended to, + and which should not. + tasks_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input tasks sequence tokens in the vocabulary of the language model. + + Indices can be obtained using [`OmDetTurboProcessor`]. See [`OmDetTurboProcessor.__call__`] for + details. + + [What are input IDs?](../glossary#input-ids) + tasks_attention_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): + Attention mask for the tasks. This is a binary mask that indicates which tokens should be attended to, + and which should not. + classes_structure (torch.LongTensor of shape `(batch_size)`): + Structure of the classes. This tensor indicates the number of classes for each task. Examples: diff --git a/src/transformers/models/oneformer/modeling_oneformer.py b/src/transformers/models/oneformer/modeling_oneformer.py index 212edc1c85..b45e4219e8 100644 --- a/src/transformers/models/oneformer/modeling_oneformer.py +++ b/src/transformers/models/oneformer/modeling_oneformer.py @@ -30,12 +30,10 @@ from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_accelerate_available, is_scipy_available, logging, - replace_return_docstrings, requires_backends, ) from ...utils.backbone_utils import load_backbone @@ -49,10 +47,6 @@ if is_accelerate_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "OneFormerConfig" -_CHECKPOINT_FOR_DOC = "shi-labs/oneformer_ade20k_swin_tiny" - - if is_scipy_available(): from scipy.optimize import linear_sum_assignment @@ -2753,41 +2747,7 @@ class OneFormerTaskModel(nn.Module): return task_tokens -ONEFORMER_START_DOCSTRING = r""" - This model is a PyTorch [nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a - regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. - - Parameters: - config ([`OneFormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ONEFORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`OneFormerProcessor`]. See - [`OneFormerProcessor.__call__`] for details. - task_inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Task inputs. Task inputs can be obtained using [`AutoImageProcessor`]. See [`OneFormerProcessor.__call__`] - for details. - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of Detr's decoder attention layers. - return_dict (`bool`, *optional*): - Whether or not to return a [`~OneFormerModelOutput`] instead of a plain tuple. -""" - - +@auto_docstring class OneFormerPreTrainedModel(PreTrainedModel): config_class = OneFormerConfig base_model_prefix = "model" @@ -2903,10 +2863,7 @@ class OneFormerPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -@add_start_docstrings( - "The bare OneFormer Model outputting raw hidden-states without any specific head on top.", - ONEFORMER_START_DOCSTRING, -) +@auto_docstring class OneFormerModel(OneFormerPreTrainedModel): main_input_name = ["pixel_values", "task_inputs"] @@ -2924,8 +2881,7 @@ class OneFormerModel(OneFormerPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(ONEFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OneFormerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -2937,8 +2893,12 @@ class OneFormerModel(OneFormerPreTrainedModel): return_dict: Optional[bool] = None, ) -> OneFormerModelOutput: r""" - Returns: - `OneFormerModelOutput` + task_inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Task inputs. Task inputs can be obtained using [`AutoImageProcessor`]. See [`OneFormerProcessor.__call__`] + for details. + text_inputs (`List[torch.Tensor]`, *optional*): + Tensor fof shape `(num_queries, sequence_length)` to be fed to a model + Example: ```python @@ -3032,9 +2992,10 @@ class OneFormerModel(OneFormerPreTrainedModel): return output -@add_start_docstrings( - "OneFormer Model for instance, semantic and panoptic image segmentation.", - ONEFORMER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + OneFormer Model for instance, semantic and panoptic image segmentation. + """ ) class OneFormerForUniversalSegmentation(OneFormerPreTrainedModel): main_input_name = ["pixel_values", "task_inputs"] @@ -3103,8 +3064,7 @@ class OneFormerForUniversalSegmentation(OneFormerPreTrainedModel): def get_loss(self, loss_dict: Dict[str, Tensor]) -> Tensor: return sum(loss_dict.values()) - @add_start_docstrings_to_model_forward(ONEFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OneFormerForUniversalSegmentationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -3119,6 +3079,9 @@ class OneFormerForUniversalSegmentation(OneFormerPreTrainedModel): return_dict: Optional[bool] = None, ) -> OneFormerForUniversalSegmentationOutput: r""" + task_inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Task inputs. Task inputs can be obtained using [`AutoImageProcessor`]. See [`OneFormerProcessor.__call__`] + for details. text_inputs (`List[torch.Tensor]`, *optional*): Tensor fof shape `(num_queries, sequence_length)` to be fed to a model mask_labels (`List[torch.Tensor]`, *optional*): @@ -3126,9 +3089,9 @@ class OneFormerForUniversalSegmentation(OneFormerPreTrainedModel): class_labels (`List[torch.LongTensor]`, *optional*): list of target class labels of shape `(num_labels, height, width)` to be fed to a model. They identify the labels of `mask_labels`, e.g. the label of `mask_labels[i][j]` if `class_labels[i][j]`. + output_auxiliary_logits (`bool`, *optional*): + Whether or not to output auxiliary logits. - Returns: - `OneFormerUniversalSegmentationOutput` Example: Universal segmentation example: diff --git a/src/transformers/models/openai/modeling_openai.py b/src/transformers/models/openai/modeling_openai.py index 244ad6d50a..55aa53c40f 100644 --- a/src/transformers/models/openai/modeling_openai.py +++ b/src/transformers/models/openai/modeling_openai.py @@ -32,20 +32,14 @@ from ...modeling_utils import PreTrainedModel from ...pytorch_utils import Conv1D, find_pruneable_heads_and_indices, prune_conv1d_layer from ...utils import ( ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_openai import OpenAIGPTConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "openai-community/openai-gpt" -_CONFIG_FOR_DOC = "OpenAIGPTConfig" - def load_tf_weights_in_openai_gpt(model, config, openai_checkpoint_folder_path): """Load tf pre-trained weights in a pytorch model (from NumPy arrays here)""" @@ -362,12 +356,8 @@ class OpenAIGPTSequenceSummary(nn.Module): return output +@auto_docstring class OpenAIGPTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = OpenAIGPTConfig load_tf_weights = load_tf_weights_in_openai_gpt base_model_prefix = "transformer" @@ -424,76 +414,7 @@ class OpenAIGPTDoubleHeadsModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -OPENAI_GPT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OpenAIGPTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -OPENAI_GPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare OpenAI GPT transformer model outputting raw hidden-states without any specific head on top.", - OPENAI_GPT_START_DOCSTRING, -) +@auto_docstring class OpenAIGPTModel(OpenAIGPTPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -520,12 +441,7 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.h[layer].attn.prune_heads(heads) - @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -618,12 +534,11 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" OpenAI GPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - OPENAI_GPT_START_DOCSTRING, + """ ) class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -642,12 +557,7 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -710,14 +620,13 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel, GenerationMixin): return {"input_ids": input_ids} -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + OpenAI GPT Model transformer with a language modeling and a multiple-choice classification head on top e.g. for + RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the + input embeddings, the classification head takes as input the input of a specified classification token index in the + input sequence). """ -OpenAI GPT Model transformer with a language modeling and a multiple-choice classification head on top e.g. for -RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the -input embeddings, the classification head takes as input the input of a specified classification token index in the -input sequence). -""", - OPENAI_GPT_START_DOCSTRING, ) class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): _tied_weights_keys = ["lm_head.weight"] @@ -739,8 +648,7 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OpenAIGPTDoubleHeadsModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -768,8 +676,6 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above) - Return: - Examples: ```python @@ -835,8 +741,8 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Original OpenAI GPT Model transformer with a sequence classification head on top (linear layer). [`OpenAIGPTForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the @@ -844,8 +750,7 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - OPENAI_GPT_START_DOCSTRING, + """ ) class OpenAIGPTForSequenceClassification(OpenAIGPTPreTrainedModel): def __init__(self, config): @@ -857,12 +762,7 @@ class OpenAIGPTForSequenceClassification(OpenAIGPTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(OPENAI_GPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index c26e46ba5a..1f115cb8a7 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -24,9 +24,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - AttentionMaskConverter, -) +from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import FlashAttentionKwargs, is_flash_attn_available from ...modeling_outputs import ( BaseModelOutputWithPast, @@ -36,16 +34,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_opt import OPTConfig @@ -61,17 +50,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/opt-350m" -_CONFIG_FOR_DOC = "OPTConfig" - -# Base model docstring -_EXPECTED_OUTPUT_SHAPE = [1, 8, 1024] - -# SequenceClassification docstring -_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "ArthurZ/opt-350m-dummy-sc" -_SEQ_CLASS_EXPECTED_LOSS = 1.71 -_SEQ_CLASS_EXPECTED_OUTPUT = "'LABEL_0'" - class OPTLearnedPositionalEmbedding(nn.Embedding): """ @@ -326,27 +304,7 @@ class OPTDecoderLayer(nn.Module): return outputs -OPT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OPTConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare OPT Model outputting raw hidden-states without any specific head on top.", - OPT_START_DOCSTRING, -) +@auto_docstring class OPTPreTrainedModel(PreTrainedModel): config_class = OPTConfig base_model_prefix = "model" @@ -375,77 +333,6 @@ class OPTPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -OPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. for padding use -1. - - [What are position IDs?](../glossary#position-ids) - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class OPTDecoder(OPTPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`OPTDecoderLayer`] @@ -836,10 +723,7 @@ class OPTDecoder(OPTPreTrainedModel): ) -@add_start_docstrings( - "The bare OPT Model outputting raw hidden-states without any specific head on top.", - OPT_START_DOCSTRING, -) +@auto_docstring class OPTModel(OPTPreTrainedModel): def __init__(self, config: OPTConfig): super().__init__(config) @@ -857,13 +741,7 @@ class OPTModel(OPTPreTrainedModel): return self.decoder @can_return_tuple - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPast, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -945,7 +823,7 @@ class OPTForCausalLM(OPTPreTrainedModel, GenerationMixin): return self.model.decoder @can_return_tuple - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -963,70 +841,10 @@ class OPTForCausalLM(OPTPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. for padding use -1. - - [What are position IDs?](../glossary#position-ids) - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1098,8 +916,8 @@ class OPTForCausalLM(OPTPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The OPT Model transformer with a sequence classification head on top (linear layer). [`OPTForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1110,8 +928,7 @@ class OPTForCausalLM(OPTPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - OPT_START_DOCSTRING, + """ ) class OPTForSequenceClassification(OPTPreTrainedModel): def __init__(self, config: OPTConfig): @@ -1123,14 +940,7 @@ class OPTForSequenceClassification(OPTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, - output_type=SequenceClassifierOutputWithPast, - config_class=_CONFIG_FOR_DOC, - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1232,13 +1042,7 @@ class OPTForSequenceClassification(OPTPreTrainedModel): self.model.decoder.embed_tokens = value -@add_start_docstrings( - """ - The OPT Model transformer with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - OPT_START_DOCSTRING, -) +@auto_docstring class OPTForQuestionAnswering(OPTPreTrainedModel): def __init__(self, config: OPTConfig): super().__init__(config) @@ -1248,8 +1052,7 @@ class OPTForQuestionAnswering(OPTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=QuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1266,17 +1069,6 @@ class OPTForQuestionAnswering(OPTPreTrainedModel): position_ids: Optional[torch.LongTensor] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - - Returns: - Example: ```python diff --git a/src/transformers/models/owlv2/modeling_owlv2.py b/src/transformers/models/owlv2/modeling_owlv2.py index 7d83ab0004..5affe9ca8d 100644 --- a/src/transformers/models/owlv2/modeling_owlv2.py +++ b/src/transformers/models/owlv2/modeling_owlv2.py @@ -26,15 +26,7 @@ from ...activations import ACT2FN from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_vision_available, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, is_vision_available, logging, torch_int from .configuration_owlv2 import Owlv2Config, Owlv2TextConfig, Owlv2VisionConfig @@ -44,7 +36,6 @@ if is_vision_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/owlv2-base-patch16-ensemble" # See all Owlv2 models at https://huggingface.co/models?filter=owlv2 @@ -556,13 +547,9 @@ class Owlv2EncoderLayer(nn.Module): return outputs +@auto_docstring # Copied from transformers.models.owlvit.modeling_owlvit.OwlViTPreTrainedModel with OwlViT->Owlv2,owlvit->owlv2 class Owlv2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Owlv2Config base_model_prefix = "owlv2" supports_gradient_checkpointing = True @@ -609,129 +596,6 @@ class Owlv2PreTrainedModel(PreTrainedModel): module.bias.data.zero_() -OWLV2_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Owvl2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -OWLV2_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, num_max_text_queries, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLV2_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLV2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_base_image_embeds (`bool`, *optional*): - Whether or not to return the base image embeddings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLV2_OBJECT_DETECTION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids). - attention_mask (`torch.Tensor` of shape `(batch_size, num_max_text_queries, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - output_hidden_states (`bool`, *optional*): - Whether or not to return the last hidden state. See `text_model_last_hidden_state` and - `vision_model_last_hidden_state` under returned tensors for more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLV2_IMAGE_GUIDED_OBJECT_DETECTION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - query_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values of query image(s) to be detected. Pass in one query image per target image. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.owlvit.modeling_owlvit.OwlViTEncoder with OwlViT->Owlv2 class Owlv2Encoder(nn.Module): """ @@ -832,8 +696,7 @@ class Owlv2TextTransformer(nn.Module): self.encoder = Owlv2Encoder(config) self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(OWLV2_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Owlv2TextConfig) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -844,7 +707,10 @@ class Owlv2TextTransformer(nn.Module): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -913,8 +779,7 @@ class Owlv2TextModel(Owlv2PreTrainedModel): def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value - @add_start_docstrings_to_model_forward(OWLV2_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Owlv2TextConfig) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -924,7 +789,10 @@ class Owlv2TextModel(Owlv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) Examples: ```python @@ -961,8 +829,7 @@ class Owlv2VisionTransformer(nn.Module): self.encoder = Owlv2Encoder(config) self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(OWLV2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Owlv2VisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -971,9 +838,6 @@ class Owlv2VisionTransformer(nn.Module): interpolate_pos_encoding: Optional[bool] = False, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1024,8 +888,7 @@ class Owlv2VisionModel(Owlv2PreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(OWLV2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Owlv2VisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1035,8 +898,6 @@ class Owlv2VisionModel(Owlv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python >>> from PIL import Image @@ -1063,7 +924,7 @@ class Owlv2VisionModel(Owlv2PreTrainedModel): ) -@add_start_docstrings(OWLV2_START_DOCSTRING) +@auto_docstring # Copied from transformers.models.owlvit.modeling_owlvit.OwlViTModel with google/owlvit-base-patch32->google/owlv2-base-patch16-ensemble, OWLVIT->OWLV2,OwlViT->Owlv2,owlvit->owlv2,OWL-ViT->OWLv2 class Owlv2Model(Owlv2PreTrainedModel): config_class = Owlv2Config @@ -1100,7 +961,7 @@ class Owlv2Model(Owlv2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(OWLV2_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1110,6 +971,11 @@ class Owlv2Model(Owlv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Owlv2TextModel`]. @@ -1135,7 +1001,7 @@ class Owlv2Model(Owlv2PreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(OWLV2_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1182,8 +1048,7 @@ class Owlv2Model(Owlv2PreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(OWLV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Owlv2Output, config_class=Owlv2Config) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1197,7 +1062,10 @@ class Owlv2Model(Owlv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Owlv2Output]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. + return_base_image_embeds (`bool`, *optional*): + Whether or not to return the base image embeddings. Examples: ```python @@ -1611,8 +1479,7 @@ class Owlv2ForObjectDetection(Owlv2PreTrainedModel): return query_embeds, box_indices, pred_boxes - @add_start_docstrings_to_model_forward(OWLV2_IMAGE_GUIDED_OBJECT_DETECTION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Owlv2ImageGuidedObjectDetectionOutput, config_class=Owlv2Config) + @auto_docstring def image_guided_detection( self, pixel_values: torch.FloatTensor, @@ -1623,7 +1490,8 @@ class Owlv2ForObjectDetection(Owlv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Owlv2ImageGuidedObjectDetectionOutput: r""" - Returns: + query_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values of query image(s) to be detected. Pass in one query image per target image. Examples: ```python @@ -1729,8 +1597,7 @@ class Owlv2ForObjectDetection(Owlv2PreTrainedModel): vision_model_output=vision_outputs, ) - @add_start_docstrings_to_model_forward(OWLV2_OBJECT_DETECTION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Owlv2ObjectDetectionOutput, config_class=Owlv2Config) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -1742,7 +1609,13 @@ class Owlv2ForObjectDetection(Owlv2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Owlv2ObjectDetectionOutput: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids). + output_hidden_states (`bool`, *optional*): + Whether or not to return the last hidden state. See `text_model_last_hidden_state` and + `vision_model_last_hidden_state` under returned tensors for more detail. Examples: ```python diff --git a/src/transformers/models/owlvit/image_processing_owlvit_fast.py b/src/transformers/models/owlvit/image_processing_owlvit_fast.py index f048f1a0da..8e8114451f 100644 --- a/src/transformers/models/owlvit/image_processing_owlvit_fast.py +++ b/src/transformers/models/owlvit/image_processing_owlvit_fast.py @@ -17,13 +17,10 @@ import warnings from typing import TYPE_CHECKING, List, Optional, Tuple, Union -from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BaseImageProcessorFast, -) +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_transforms import center_to_corners_format from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling -from ...utils import TensorType, add_start_docstrings, is_torch_available, logging +from ...utils import TensorType, auto_docstring, is_torch_available, logging if TYPE_CHECKING: @@ -39,10 +36,7 @@ if is_torch_available(): logger = logging.get_logger(__name__) -@add_start_docstrings( - "Constructs a fast OwlViT image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class OwlViTImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = OPENAI_CLIP_MEAN diff --git a/src/transformers/models/owlvit/modeling_owlvit.py b/src/transformers/models/owlvit/modeling_owlvit.py index 2eb0114cf1..eee1333462 100644 --- a/src/transformers/models/owlvit/modeling_owlvit.py +++ b/src/transformers/models/owlvit/modeling_owlvit.py @@ -26,15 +26,7 @@ from ...activations import ACT2FN from ...modeling_attn_mask_utils import _create_4d_causal_attention_mask, _prepare_4d_attention_mask from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_vision_available, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, is_vision_available, logging, torch_int from .configuration_owlvit import OwlViTConfig, OwlViTTextConfig, OwlViTVisionConfig @@ -44,7 +36,6 @@ if is_vision_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/owlvit-base-patch32" # See all OwlViT models at https://huggingface.co/models?filter=owlvit @@ -544,12 +535,8 @@ class OwlViTEncoderLayer(nn.Module): return outputs +@auto_docstring class OwlViTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = OwlViTConfig base_model_prefix = "owlvit" supports_gradient_checkpointing = True @@ -596,127 +583,6 @@ class OwlViTPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -OWLVIT_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OwlViTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -OWLVIT_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, num_max_text_queries, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLVIT_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLVIT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLVIT_OBJECT_DETECTION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids). - attention_mask (`torch.Tensor` of shape `(batch_size, num_max_text_queries, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - output_hidden_states (`bool`, *optional*): - Whether or not to return the last hidden state. See `text_model_last_hidden_state` and - `vision_model_last_hidden_state` under returned tensors for more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OWLVIT_IMAGE_GUIDED_OBJECT_DETECTION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - query_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values of query image(s) to be detected. Pass in one query image per target image. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class OwlViTEncoder(nn.Module): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a @@ -815,8 +681,7 @@ class OwlViTTextTransformer(nn.Module): self.encoder = OwlViTEncoder(config) self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(OWLVIT_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=OwlViTTextConfig) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -827,7 +692,10 @@ class OwlViTTextTransformer(nn.Module): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -895,8 +763,7 @@ class OwlViTTextModel(OwlViTPreTrainedModel): def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value - @add_start_docstrings_to_model_forward(OWLVIT_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=OwlViTTextConfig) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -906,7 +773,10 @@ class OwlViTTextModel(OwlViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) Examples: ```python @@ -942,8 +812,7 @@ class OwlViTVisionTransformer(nn.Module): self.encoder = OwlViTEncoder(config) self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(OWLVIT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=OwlViTVisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -952,9 +821,6 @@ class OwlViTVisionTransformer(nn.Module): interpolate_pos_encoding: Optional[bool] = False, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1004,8 +870,7 @@ class OwlViTVisionModel(OwlViTPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(OWLVIT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=OwlViTVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1015,8 +880,6 @@ class OwlViTVisionModel(OwlViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python >>> from PIL import Image @@ -1043,7 +906,7 @@ class OwlViTVisionModel(OwlViTPreTrainedModel): ) -@add_start_docstrings(OWLVIT_START_DOCSTRING) +@auto_docstring class OwlViTModel(OwlViTPreTrainedModel): config_class = OwlViTConfig @@ -1079,7 +942,7 @@ class OwlViTModel(OwlViTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(OWLVIT_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1089,6 +952,11 @@ class OwlViTModel(OwlViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`OwlViTTextModel`]. @@ -1114,7 +982,7 @@ class OwlViTModel(OwlViTPreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(OWLVIT_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1161,8 +1029,7 @@ class OwlViTModel(OwlViTPreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(OWLVIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OwlViTOutput, config_class=OwlViTConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1176,7 +1043,10 @@ class OwlViTModel(OwlViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, OwlViTOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. + return_base_image_embeds (`bool`, *optional*): + Whether or not to return the base image embeddings. Examples: ```python @@ -1566,8 +1436,7 @@ class OwlViTForObjectDetection(OwlViTPreTrainedModel): return query_embeds, box_indices, pred_boxes - @add_start_docstrings_to_model_forward(OWLVIT_IMAGE_GUIDED_OBJECT_DETECTION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OwlViTImageGuidedObjectDetectionOutput, config_class=OwlViTConfig) + @auto_docstring def image_guided_detection( self, pixel_values: torch.FloatTensor, @@ -1578,7 +1447,8 @@ class OwlViTForObjectDetection(OwlViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> OwlViTImageGuidedObjectDetectionOutput: r""" - Returns: + query_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values of query image(s) to be detected. Pass in one query image per target image. Examples: ```python @@ -1669,8 +1539,7 @@ class OwlViTForObjectDetection(OwlViTPreTrainedModel): vision_model_output=vision_outputs, ) - @add_start_docstrings_to_model_forward(OWLVIT_OBJECT_DETECTION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=OwlViTObjectDetectionOutput, config_class=OwlViTConfig) + @auto_docstring def forward( self, input_ids: torch.Tensor, @@ -1682,7 +1551,13 @@ class OwlViTForObjectDetection(OwlViTPreTrainedModel): return_dict: Optional[bool] = None, ) -> OwlViTObjectDetectionOutput: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size * num_max_text_queries, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids). + output_hidden_states (`bool`, *optional*): + Whether or not to return the last hidden state. See `text_model_last_hidden_state` and + `vision_model_last_hidden_state` under returned tensors for more detail. Examples: ```python diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index d8096d8113..b584a3b131 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -27,24 +27,13 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, ModelOutput, auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging from ..auto import AutoModel from .configuration_paligemma import PaliGemmaConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "PaliGemmaConfig" - @dataclass class PaligemmaModelOutputWithPast(BaseModelOutputWithPast): @@ -130,27 +119,7 @@ class PaliGemmaMultiModalProjector(nn.Module): return hidden_states -PALIGEMMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PaliGemmaConfig`] or [`PaliGemmaVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", - PALIGEMMA_START_DOCSTRING, -) +@auto_docstring class PaliGemmaPreTrainedModel(PreTrainedModel): config_class = PaliGemmaConfig base_model_prefix = "" @@ -175,79 +144,10 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -PALIGEMMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`PaliGemmaProcessor`] uses - [`SiglipImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """Base Paligemma model which consists of a vision backbone and a language model withou language modeling head.""", - PALIGEMMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Base Paligemma model which consists of a vision backbone and a language model withou language modeling head., + """ ) class PaliGemmaModel(PaliGemmaPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -357,7 +257,7 @@ class PaliGemmaModel(PaliGemmaPreTrainedModel): return image_features @can_return_tuple - @add_start_docstrings_to_model_forward(PALIGEMMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -375,6 +275,34 @@ class PaliGemmaModel(PaliGemmaPreTrainedModel): return_dict: Optional[bool] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, PaligemmaModelOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, PaliGemmaForConditionalGeneration + + >>> model = PaliGemmaForConditionalGeneration.from_pretrained("google/paligemma2-3b-mix-224") + >>> processor = AutoProcessor.from_pretrained("google/paligemma2-3b-mix-224") + + >>> prompt = "Where is the cat standing?" + >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, text=prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(**inputs,) + >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Where is the cat standing?\nsnow" + ```""" + if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError("You must specify exactly one of input_ids or inputs_embeds") @@ -456,9 +384,10 @@ class PaliGemmaModel(PaliGemmaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """Base Paligemma model which consists of a vision backbone and a language model withou language modeling head.""", - PALIGEMMA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Base Paligemma model which consists of a vision backbone and a language model without language modeling head., + """ ) class PaliGemmaForConditionalGeneration(PaliGemmaPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -501,8 +430,7 @@ class PaliGemmaForConditionalGeneration(PaliGemmaPreTrainedModel, GenerationMixi return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(PALIGEMMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=PaliGemmaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -522,19 +450,10 @@ class PaliGemmaForConditionalGeneration(PaliGemmaPreTrainedModel, GenerationMixi **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, PaliGemmaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. Example: diff --git a/src/transformers/models/patchtsmixer/modeling_patchtsmixer.py b/src/transformers/models/patchtsmixer/modeling_patchtsmixer.py index 2336338089..983961f3b4 100644 --- a/src/transformers/models/patchtsmixer/modeling_patchtsmixer.py +++ b/src/transformers/models/patchtsmixer/modeling_patchtsmixer.py @@ -25,56 +25,12 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput from ...time_series_utils import NegativeBinomialOutput, NormalOutput, StudentTOutput -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_patchtsmixer import PatchTSMixerConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "PatchTSMixerConfig" - - -PATCHTSMIXER_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PatchTSMixerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. - mask_input (`bool`, *optional*, defaults to `False`): - If True, Masking will be enabled. False otherwise. -""" - -PATCHTSMIXER_INPUTS_DOCSTRING = r""" - Args: - past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): - Context values of the time series. For a pretraining task, this denotes the input time series to predict - the masked portion. For a forecasting task, this denotes the history/past time series values. Similarly, - for classification or regression tasks, it denotes the appropriate context values of the time series. - - For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, it is - greater than 1. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - class PatchTSMixerGatedAttention(nn.Module): """ @@ -757,6 +713,7 @@ class PatchTSMixerLinearHead(nn.Module): return hidden_features +@auto_docstring class PatchTSMixerPreTrainedModel(PreTrainedModel): # Weight initialization config_class = PatchTSMixerConfig @@ -1210,7 +1167,7 @@ class PatchTSMixerEncoder(PatchTSMixerPreTrainedModel): if config.post_init: self.post_init() - @replace_return_docstrings(output_type=PatchTSMixerEncoderOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1218,21 +1175,14 @@ class PatchTSMixerEncoder(PatchTSMixerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, PatchTSMixerEncoderOutput]: r""" - Args: - past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): - Context values of the time series. For a pretraining task, this denotes the input time series to - predict the masked portion. For a forecasting task, this denotes the history/past time series values. - Similarly, for classification or regression tasks, it denotes the appropriate context values of the - time series. + past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): + Context values of the time series. For a pretraining task, this denotes the input time series to + predict the masked portion. For a forecasting task, this denotes the history/past time series values. + Similarly, for classification or regression tasks, it denotes the appropriate context values of the + time series. - For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, - it is greater than 1. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, + it is greater than 1. Returns: `torch.FloatTensor` of shape `(batch_size, n_vars, num_patches, d_model)` @@ -1291,12 +1241,17 @@ class PatchTSMixerModelOutput(ModelOutput): scale: Optional[torch.FloatTensor] = None -@add_start_docstrings( - "The PatchTSMixer Model for time-series forecasting.", - PATCHTSMIXER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PatchTSMixer Model for time-series forecasting. + """ ) class PatchTSMixerModel(PatchTSMixerPreTrainedModel): def __init__(self, config: PatchTSMixerConfig, mask_input: bool = False): + r""" + mask_input (bool, *optional*, defaults to `False`): + Whether to mask the input using the [`PatchTSMixerMasking`] module. + """ super().__init__(config) self.use_return_dict = config.use_return_dict @@ -1319,8 +1274,7 @@ class PatchTSMixerModel(PatchTSMixerPreTrainedModel): if config.post_init: self.post_init() - @add_start_docstrings_to_model_forward(PATCHTSMIXER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=PatchTSMixerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1329,14 +1283,18 @@ class PatchTSMixerModel(PatchTSMixerPreTrainedModel): return_dict: Optional[bool] = None, ) -> PatchTSMixerModelOutput: r""" + past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): + Context values of the time series. For a pretraining task, this denotes the input time series to predict + the masked portion. For a forecasting task, this denotes the history/past time series values. Similarly, + for classification or regression tasks, it denotes the appropriate context values of the time series. + + For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, it is + greater than 1. observed_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in `[0, 1]`: - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - - Returns: - + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). """ return_dict = return_dict if return_dict is not None else self.use_return_dict @@ -1430,8 +1388,7 @@ class PatchTSMixerForPretraining(PatchTSMixerPreTrainedModel): if config.post_init: self.post_init() - @add_start_docstrings_to_model_forward(PATCHTSMIXER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=PatchTSMixerForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1441,16 +1398,20 @@ class PatchTSMixerForPretraining(PatchTSMixerPreTrainedModel): return_dict: Optional[bool] = None, ) -> PatchTSMixerForPreTrainingOutput: r""" + past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): + Context values of the time series. For a pretraining task, this denotes the input time series to predict + the masked portion. For a forecasting task, this denotes the history/past time series values. Similarly, + for classification or regression tasks, it denotes the appropriate context values of the time series. + + For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, it is + greater than 1. observed_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in `[0, 1]`: - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). return_loss (`bool`, *optional*): Whether to return the loss in the `forward` call. - - Returns: - """ return_dict = return_dict if return_dict is not None else self.use_return_dict @@ -1633,8 +1594,7 @@ class PatchTSMixerForPrediction(PatchTSMixerPreTrainedModel): if config.post_init: self.post_init() - @add_start_docstrings_to_model_forward(PATCHTSMIXER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=PatchTSMixerForPredictionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1645,14 +1605,21 @@ class PatchTSMixerForPrediction(PatchTSMixerPreTrainedModel): return_dict: Optional[bool] = None, ) -> PatchTSMixerForPredictionOutput: r""" + past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): + Context values of the time series. For a pretraining task, this denotes the input time series to predict + the masked portion. For a forecasting task, this denotes the history/past time series values. Similarly, + for classification or regression tasks, it denotes the appropriate context values of the time series. + + For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, it is + greater than 1. observed_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in `[0, 1]`: - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). future_values (`torch.FloatTensor` of shape `(batch_size, target_len, num_input_channels)` for forecasting,: - `(batch_size, num_targets)` for regression, or `(batch_size,)` for classification, *optional*): Target - values of the time series, that serve as labels for the model. The `future_values` is what the + `(batch_size, num_targets)` for regression, or `(batch_size,)` for classification, *optional*): + Target values of the time series, that serve as labels for the model. The `future_values` is what the Transformer needs during training to learn to output, given the `past_values`. Note that, this is NOT required for a pretraining task. @@ -1662,9 +1629,6 @@ class PatchTSMixerForPrediction(PatchTSMixerPreTrainedModel): manually applied before the loss computation. return_loss (`bool`, *optional*): Whether to return the loss in the `forward` call. - - Returns: - """ if self.loss == "mse": loss = nn.MSELoss(reduction="mean") @@ -1851,11 +1815,7 @@ class PatchTSMixerForTimeSeriesClassification(PatchTSMixerPreTrainedModel): if config.post_init: self.post_init() - @add_start_docstrings_to_model_forward(PATCHTSMIXER_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=PatchTSMixerForTimeSeriesClassificationOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1865,8 +1825,16 @@ class PatchTSMixerForTimeSeriesClassification(PatchTSMixerPreTrainedModel): return_dict: Optional[bool] = None, ) -> PatchTSMixerForTimeSeriesClassificationOutput: r""" + past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): + Context values of the time series. For a pretraining task, this denotes the input time series to predict + the masked portion. For a forecasting task, this denotes the history/past time series values. Similarly, + for classification or regression tasks, it denotes the appropriate context values of the time series. + + For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, it is + greater than 1. target_values (`torch.FloatTensor` of shape `(batch_size, target_len, num_input_channels)` for forecasting, - `(batch_size, num_targets)` for regression, or `(batch_size,)` for classification, *optional*): Target + `(batch_size, num_targets)` for regression, or `(batch_size,)` for classification, *optional*): + Target values of the time series, that serve as labels for the model. The `target_values` is what the Transformer needs during training to learn to output, given the `past_values`. Note that, this is NOT required for a pretraining task. @@ -1881,9 +1849,6 @@ class PatchTSMixerForTimeSeriesClassification(PatchTSMixerPreTrainedModel): For a regression task, it has a shape of `(batch_size, num_targets)`. return_loss (`bool`, *optional*): Whether to return the loss in the `forward` call. - - Returns: - """ loss = torch.nn.CrossEntropyLoss() @@ -2044,8 +2009,7 @@ class PatchTSMixerForRegression(PatchTSMixerPreTrainedModel): if config.post_init: self.post_init() - @add_start_docstrings_to_model_forward(PATCHTSMIXER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=PatchTSMixerForRegressionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -2055,9 +2019,16 @@ class PatchTSMixerForRegression(PatchTSMixerPreTrainedModel): return_dict: Optional[bool] = None, ) -> PatchTSMixerForRegressionOutput: r""" + past_values (`torch.FloatTensor` of shape `(batch_size, seq_length, num_input_channels)`): + Context values of the time series. For a pretraining task, this denotes the input time series to predict + the masked portion. For a forecasting task, this denotes the history/past time series values. Similarly, + for classification or regression tasks, it denotes the appropriate context values of the time series. + + For univariate time series, `num_input_channels` dimension should be 1. For multivariate time series, it is + greater than 1. target_values (`torch.FloatTensor` of shape `(batch_size, target_len, num_input_channels)` for forecasting, - `(batch_size, num_targets)` for regression, or `(batch_size,)` for classification, *optional*): Target - values of the time series, that serve as labels for the model. The `target_values` is what the + `(batch_size, num_targets)` for regression, or `(batch_size,)` for classification, *optional*): + Target values of the time series, that serve as labels for the model. The `target_values` is what the Transformer needs during training to learn to output, given the `past_values`. Note that, this is NOT required for a pretraining task. @@ -2071,9 +2042,6 @@ class PatchTSMixerForRegression(PatchTSMixerPreTrainedModel): For a regression task, it has a shape of `(batch_size, num_targets)`. return_loss (`bool`, *optional*): Whether to return the loss in the `forward` call. - - Returns: - """ if self.loss == "mse": diff --git a/src/transformers/models/patchtst/modeling_patchtst.py b/src/transformers/models/patchtst/modeling_patchtst.py index 7ee66bec70..e5785974be 100755 --- a/src/transformers/models/patchtst/modeling_patchtst.py +++ b/src/transformers/models/patchtst/modeling_patchtst.py @@ -25,14 +25,12 @@ from ...activations import ACT2CLS from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel from ...time_series_utils import NegativeBinomialOutput, NormalOutput, StudentTOutput -from ...utils import ModelOutput, add_start_docstrings, logging +from ...utils import ModelOutput, auto_docstring, logging from .configuration_patchtst import PatchTSTConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "PatchTSTConfig" - # Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->PatchTST class PatchTSTAttention(nn.Module): @@ -587,6 +585,7 @@ class PatchTSTEncoderLayer(nn.Module): return outputs +@auto_docstring class PatchTSTPreTrainedModel(PreTrainedModel): config_class = PatchTSTConfig base_model_prefix = "model" @@ -774,23 +773,6 @@ class PatchTSTEncoder(PatchTSTPreTrainedModel): return BaseModelOutput(last_hidden_state=hidden_state, hidden_states=encoder_states, attentions=all_attentions) -PATCHTST_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PatchTSTConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - @dataclass class PatchTSTModelOutput(ModelOutput): """ @@ -1139,10 +1121,7 @@ class PatchTSTScaler(nn.Module): return data, loc, scale -@add_start_docstrings( - "The bare PatchTST Model outputting raw hidden-states without any specific head.", - PATCHTST_START_DOCSTRING, -) +@auto_docstring class PatchTSTModel(PatchTSTPreTrainedModel): def __init__(self, config: PatchTSTConfig): super().__init__(config) @@ -1283,9 +1262,10 @@ class PatchTSTMaskPretrainHead(nn.Module): return embedding -@add_start_docstrings( - "The PatchTST for pretrain model.", - PATCHTST_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PatchTST for pretrain model. + """ ) class PatchTSTForPretraining(PatchTSTPreTrainedModel): def __init__(self, config: PatchTSTConfig): @@ -1435,9 +1415,10 @@ class PatchTSTClassificationHead(nn.Module): return output -@add_start_docstrings( - "The PatchTST for classification model.", - PATCHTST_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PatchTST for classification model. + """ ) class PatchTSTForClassification(PatchTSTPreTrainedModel): def __init__(self, config: PatchTSTConfig): @@ -1454,6 +1435,7 @@ class PatchTSTForClassification(PatchTSTPreTrainedModel): # Initialize weights and apply final processing self.post_init() + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1464,27 +1446,16 @@ class PatchTSTForClassification(PatchTSTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[tuple, PatchTSTForClassificationOutput]: r""" - Parameters: - past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*): - Input sequence to the model - target_values (`torch.Tensor`, *optional*): - Labels associates with the `past_values` - past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): - Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected - in `[0, 1]`: + past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*): + Input sequence to the model + target_values (`torch.Tensor`, *optional*): + Labels associates with the `past_values` + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected + in `[0, 1]`: - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers - output_attentions (`bool`, *optional*): - Whether or not to return the output attention of all layers - return_dict (`bool`, *optional*): - Whether or not to return a `ModelOutput` instead of a plain tuple. - - Returns: - `PatchTSTForClassificationOutput` or tuple of `torch.Tensor` (if `return_dict`=False or - `config.return_dict`=False) + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). Examples: @@ -1536,12 +1507,19 @@ class PatchTSTForClassification(PatchTSTPreTrainedModel): ) -@add_start_docstrings( - "The PatchTST for regression Model.", - PATCHTST_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PatchTST for regression Model. + """ ) class PatchTSTPredictionHead(nn.Module): - def __init__(self, config: PatchTSTConfig, num_patches, distribution_output=None): + def __init__(self, config: PatchTSTConfig, num_patches: int, distribution_output=None): + r""" + num_patches (`int`): + The number of patches in the input sequence. + distribution_output (`DistributionOutput`, *optional*): + The distribution output layer for probabilistic forecasting. If None, a linear output layer is used. + """ super().__init__() self.share_projection = config.share_projection @@ -1630,9 +1608,10 @@ class PatchTSTPredictionHead(nn.Module): return output -@add_start_docstrings( - "The PatchTST for prediction model.", - PATCHTST_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PatchTST for prediction model. + """ ) class PatchTSTForPrediction(PatchTSTPreTrainedModel): def __init__(self, config: PatchTSTConfig): @@ -1876,9 +1855,10 @@ class PatchTSTRegressionHead(nn.Module): return output -@add_start_docstrings( - "The PatchTST for regression model.", - PATCHTST_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PatchTST for regression model. + """ ) class PatchTSTForRegression(PatchTSTPreTrainedModel): def __init__(self, config: PatchTSTConfig): @@ -1907,6 +1887,7 @@ class PatchTSTForRegression(PatchTSTPreTrainedModel): # Initialize weights and apply final processing self.post_init() + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1917,27 +1898,17 @@ class PatchTSTForRegression(PatchTSTPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[tuple, PatchTSTForRegressionOutput]: r""" - Parameters: - past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*): - Input sequence to the model - target_values (`torch.Tensor` of shape `(bs, num_input_channels)`): - Target values associates with the `past_values` - past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): - Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected - in `[0, 1]`: + past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*): + Input sequence to the model + target_values (`torch.Tensor` of shape `(bs, num_input_channels)`): + Target values associates with the `past_values` + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected + in `[0, 1]`: - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers - output_attentions (`bool`, *optional*): - Whether or not to return the output attention of all layers - return_dict (`bool`, *optional*): - Whether or not to return a `ModelOutput` instead of a plain tuple. - - Returns: - `PatchTSTForRegressionOutput` or tuple of `torch.Tensor` (if `return_dict`=False or - `config.return_dict`=False) + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + Whether or not to return a `ModelOutput` instead of a plain tuple. Examples: diff --git a/src/transformers/models/pegasus/modeling_pegasus.py b/src/transformers/models/pegasus/modeling_pegasus.py index b4c5a0acdc..9d0eafa9c9 100755 --- a/src/transformers/models/pegasus/modeling_pegasus.py +++ b/src/transformers/models/pegasus/modeling_pegasus.py @@ -35,21 +35,12 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_pegasus import PegasusConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/pegasus-large" -_CONFIG_FOR_DOC = "PegasusConfig" - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -452,6 +443,7 @@ class PegasusDecoderLayer(nn.Module): return outputs +@auto_docstring class PegasusPreTrainedModel(PreTrainedModel): config_class = PegasusConfig base_model_prefix = "model" @@ -471,136 +463,6 @@ class PegasusPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -PEGASUS_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PegasusConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PEGASUS_GENERATION_EXAMPLE = r""" - Summarization example: - - ```python - >>> from transformers import AutoTokenizer, PegasusForConditionalGeneration - - >>> model = PegasusForConditionalGeneration.from_pretrained("google/pegasus-xsum") - >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-xsum") - - >>> ARTICLE_TO_SUMMARIZE = ( - ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " - ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " - ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." - ... ) - >>> inputs = tokenizer(ARTICLE_TO_SUMMARIZE, max_length=1024, return_tensors="pt") - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"]) - >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "California's largest electricity provider has turned off power to hundreds of thousands of customers." - ``` -""" - -PEGASUS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Pegasus uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class PegasusEncoder(PegasusPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1087,10 +949,7 @@ class PegasusDecoder(PegasusPreTrainedModel): ) -@add_start_docstrings( - "The bare PEGASUS Model outputting raw hidden-states without any specific head on top.", - PEGASUS_START_DOCSTRING, -) +@auto_docstring class PegasusModel(PegasusPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1143,8 +1002,7 @@ class PegasusModel(PegasusPreTrainedModel): """ return (self.encoder.get_position_embeddings(), self.decoder.get_position_embeddings()) - @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1164,7 +1022,26 @@ class PegasusModel(PegasusPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqModelOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Pegasus uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1239,8 +1116,10 @@ class PegasusModel(PegasusPreTrainedModel): ) -@add_start_docstrings( - "The PEGASUS Model with a language modeling head. Can be used for summarization.", PEGASUS_START_DOCSTRING +@auto_docstring( + custom_intro=""" + The PEGASUS Model with a language modeling head. Can be used for summarization. + """ ) class PegasusForConditionalGeneration(PegasusPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1307,9 +1186,7 @@ class PegasusForConditionalGeneration(PegasusPreTrainedModel, GenerationMixin): """ return (self.model.encoder.get_position_embeddings(), self.model.decoder.get_position_embeddings()) - @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(PEGASUS_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1330,13 +1207,51 @@ class PegasusForConditionalGeneration(PegasusPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Pegasus uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Summarization: + ```python + >>> from transformers import AutoTokenizer, PegasusForConditionalGeneration + + >>> model = PegasusForConditionalGeneration.from_pretrained("google/pegasus-xsum") + >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-xsum") + + >>> ARTICLE_TO_SUMMARIZE = ( + ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " + ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " + ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." + ... ) + >>> inputs = tokenizer(ARTICLE_TO_SUMMARIZE, max_length=1024, return_tensors="pt") + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"]) + >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "California's largest electricity provider has turned off power to hundreds of thousands of customers." + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1474,7 +1389,7 @@ class PegasusForCausalLM(PegasusPreTrainedModel, GenerationMixin): self.config.max_position_embeddings = new_num_position_embeddings self.model.decoder.resize_position_embeddings(new_num_position_embeddings) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Copied from transformers.models.bart.modeling_bart.BartForCausalLM.forward with Bart->Pegasus, facebook/bart-base->google/pegasus-large def forward( self, @@ -1493,72 +1408,15 @@ class PegasusForCausalLM(PegasusPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/pegasus_x/modeling_pegasus_x.py b/src/transformers/models/pegasus_x/modeling_pegasus_x.py index 646ab19594..9656b3afe7 100755 --- a/src/transformers/models/pegasus_x/modeling_pegasus_x.py +++ b/src/transformers/models/pegasus_x/modeling_pegasus_x.py @@ -34,21 +34,12 @@ from ...modeling_outputs import ( Seq2SeqModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_pegasus_x import PegasusXConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/pegasus-x-base" -_CONFIG_FOR_DOC = "PegasusXConfig" - @dataclasses.dataclass class DimensionInfo: @@ -745,6 +736,7 @@ class PegasusXDecoderLayer(nn.Module): return outputs +@auto_docstring class PegasusXPreTrainedModel(PreTrainedModel): config_class = PegasusXConfig base_model_prefix = "model" @@ -761,120 +753,6 @@ class PegasusXPreTrainedModel(PreTrainedModel): module.weight.data.normal_(mean=0.0, std=std) -PEGASUS_X_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PegasusXConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PEGASUS_X_GENERATION_EXAMPLE = r""" - Summarization example: - - ```python - >>> from transformers import AutoTokenizer, PegasusXForConditionalGeneration - - >>> model = PegasusXForConditionalGeneration.from_pretrained("google/pegasus-x-base") - >>> tokenizer = AutoTokenizer.from_pretrained("google/pegasus-x-large") - - >>> ARTICLE_TO_SUMMARIZE = ( - ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " - ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " - ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." - ... ) - >>> inputs = tokenizer(ARTICLE_TO_SUMMARIZE, max_length=1024, return_tensors="pt") - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"]) - >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - "California's largest electricity provider has turned off power to hundreds of thousands of customers." - ``` -""" - -PEGASUS_X_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - PEGASUS-X uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class PegasusXEncoder(PegasusXPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1314,10 +1192,7 @@ class PegasusXDecoder(PegasusXPreTrainedModel): ) -@add_start_docstrings( - "The bare PEGASUS-X Model outputting raw hidden-states without any specific head on top.", - PEGASUS_X_START_DOCSTRING, -) +@auto_docstring class PegasusXModel(PegasusXPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1374,8 +1249,7 @@ class PegasusXModel(PegasusXPreTrainedModel): """ return (self.encoder.get_position_embeddings(), self.decoder.get_position_embeddings()) - @add_start_docstrings_to_model_forward(PEGASUS_X_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1392,7 +1266,20 @@ class PegasusXModel(PegasusXPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqModelOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + PEGASUS-X uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. Example: @@ -1464,7 +1351,11 @@ class PegasusXModel(PegasusXPreTrainedModel): ) -@add_start_docstrings("The PEGASUS-X for conditional generation (e.g. summarization).", PEGASUS_X_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + The PEGASUS-X for conditional generation (e.g. summarization). + """ +) class PegasusXForConditionalGeneration(PegasusXPreTrainedModel, GenerationMixin): base_model_prefix = "model" _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"] @@ -1512,9 +1403,7 @@ class PegasusXForConditionalGeneration(PegasusXPreTrainedModel, GenerationMixin) """ return (self.model.encoder.get_position_embeddings(), self.model.decoder.get_position_embeddings()) - @add_start_docstrings_to_model_forward(PEGASUS_X_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(PEGASUS_X_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1532,13 +1421,24 @@ class PegasusXForConditionalGeneration(PegasusXPreTrainedModel, GenerationMixin) return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + PEGASUS-X uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/perceiver/image_processing_perceiver_fast.py b/src/transformers/models/perceiver/image_processing_perceiver_fast.py index 87b24e7684..3dcdd1ffcb 100644 --- a/src/transformers/models/perceiver/image_processing_perceiver_fast.py +++ b/src/transformers/models/perceiver/image_processing_perceiver_fast.py @@ -16,12 +16,12 @@ from typing import Optional, Union -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, BatchFeature +from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature from ...image_transforms import group_images_by_shape, reorder_images from ...image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling, SizeDict from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -38,10 +38,7 @@ if is_torchvision_available(): from torchvision.transforms import functional as F -@add_start_docstrings( - "Constructs a fast Perceiver image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class PerceiverImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = IMAGENET_DEFAULT_MEAN diff --git a/src/transformers/models/perceiver/modeling_perceiver.py b/src/transformers/models/perceiver/modeling_perceiver.py index 047fd5b6ba..14c969a412 100755 --- a/src/transformers/models/perceiver/modeling_perceiver.py +++ b/src/transformers/models/perceiver/modeling_perceiver.py @@ -31,14 +31,7 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, meshgrid, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_perceiver import PerceiverConfig @@ -49,9 +42,6 @@ PostprocessorType = Callable[..., Any] logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "deepmind/language-perceiver" -_CONFIG_FOR_DOC = "PerceiverConfig" - @dataclass class PerceiverModelOutput(ModelOutput): @@ -601,12 +591,8 @@ class PerceiverEncoder(nn.Module): ) +@auto_docstring class PerceiverPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = PerceiverConfig base_model_prefix = "perceiver" main_input_name = "inputs" @@ -635,80 +621,9 @@ class PerceiverPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -PERCEIVER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`PerceiverConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PERCEIVER_MODEL_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`PerceiverConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. - decoder (*DecoderType*, *optional*): - Optional decoder to use to decode the latent representation of the encoder. Examples include - *transformers.models.perceiver.modeling_perceiver.PerceiverBasicDecoder*, - *transformers.models.perceiver.modeling_perceiver.PerceiverClassificationDecoder*, - *transformers.models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder*. - input_preprocessor (*PreprocessorType*, *optional*): - Optional input preprocessor to use. Examples include - *transformers.models.perceiver.modeling_perceiver.PerceiverImagePreprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverAudioPreprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverTextPreprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverMultimodalPreprocessor*. - output_postprocessor (*PostprocessorType*, *optional*): - Optional output postprocessor to use. Examples include - *transformers.models.perceiver.modeling_perceiver.PerceiverImagePostprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverAudioPostprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverClassificationPostprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverProjectionPostprocessor*, - *transformers.models.perceiver.modeling_perceiver.PerceiverMultimodalPostprocessor*. - - Note that you can define your own decoders, preprocessors and/or postprocessors to fit your use-case. -""" - -PERCEIVER_INPUTS_DOCSTRING = r""" - Args: - inputs (`torch.FloatTensor`): - Inputs to the perceiver. Can be anything: images, text, audio, video, etc. - attention_mask (`torch.FloatTensor` of shape `{0}`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """The Perceiver: a scalable, fully attentional architecture. +@auto_docstring( + custom_intro=""" + The Perceiver: a scalable, fully attentional architecture. @@ -717,17 +632,24 @@ PERCEIVER_INPUTS_DOCSTRING = r""" position embeddings to the higher resolution. - """, - PERCEIVER_MODEL_START_DOCSTRING, + """ ) class PerceiverModel(PerceiverPreTrainedModel): def __init__( self, config, - decoder=None, + decoder: Optional["PerceiverAbstractDecoder"] = None, input_preprocessor: PreprocessorType = None, output_postprocessor: PostprocessorType = None, ): + r""" + decoder (`PerceiverDecoder`, *optional*): + Decoder module that transforms latent representations into task predictions. + input_preprocessor (`PreprocessorType`, *optional*): + Preprocessor that encodes raw inputs into tensors for the model. + output_postprocessor (`PostprocessorType`, *optional*): + Postprocessor that transforms model outputs into final predictions. + """ super().__init__(config) self.config = config @@ -756,8 +678,7 @@ class PerceiverModel(PerceiverPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) - @replace_return_docstrings(output_type=PerceiverModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: torch.FloatTensor, @@ -770,7 +691,11 @@ class PerceiverModel(PerceiverPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, PerceiverModelOutput]: r""" - Returns: + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. + subsampled_output_points (`Dict[str, torch.Tensor]`, *optional*): + Dictionary of tensors used as queries for the decoder. The decoder maps these queries to the latent + representation of the model. Used for subsampled decoding, e.g. when only decoding certain image patches. Examples: @@ -960,7 +885,11 @@ class PerceiverModel(PerceiverPreTrainedModel): ) -@add_start_docstrings("""Example use of Perceiver for masked language modeling.""", PERCEIVER_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + Example use of Perceiver for masked language modeling. + """ +) class PerceiverForMaskedLM(PerceiverPreTrainedModel): def __init__(self, config: PerceiverConfig): super().__init__(config) @@ -993,8 +922,7 @@ class PerceiverForMaskedLM(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverMaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1007,13 +935,13 @@ class PerceiverForMaskedLM(PerceiverPreTrainedModel): input_ids: Optional[torch.Tensor] = None, ) -> Union[Tuple, PerceiverMaskedLMOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -1095,7 +1023,11 @@ class PerceiverForMaskedLM(PerceiverPreTrainedModel): ) -@add_start_docstrings("""Example use of Perceiver for text classification.""", PERCEIVER_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + Example use of Perceiver for text classification. + """ +) class PerceiverForSequenceClassification(PerceiverPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1117,8 +1049,7 @@ class PerceiverForSequenceClassification(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1131,13 +1062,13 @@ class PerceiverForSequenceClassification(PerceiverPreTrainedModel): input_ids: Optional[torch.Tensor] = None, ) -> Union[Tuple, PerceiverClassifierOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1207,19 +1138,18 @@ class PerceiverForSequenceClassification(PerceiverPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Example use of Perceiver for image classification, for tasks such as ImageNet. + + This model uses learned position embeddings. In other words, this model is not given any privileged information about + the structure of images. As shown in the paper, this model can achieve a top-1 accuracy of 72.7 on ImageNet. + + [`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] + (with `prep_type="conv1x1"`) to preprocess the input images, and + [`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of + [`PerceiverModel`] into classification logits. """ -Example use of Perceiver for image classification, for tasks such as ImageNet. - -This model uses learned position embeddings. In other words, this model is not given any privileged information about -the structure of images. As shown in the paper, this model can achieve a top-1 accuracy of 72.7 on ImageNet. - -[`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] -(with `prep_type="conv1x1"`) to preprocess the input images, and -[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of -[`PerceiverModel`] into classification logits. -""", - PERCEIVER_START_DOCSTRING, ) class PerceiverForImageClassificationLearned(PerceiverPreTrainedModel): def __init__(self, config): @@ -1252,8 +1182,7 @@ class PerceiverForImageClassificationLearned(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1267,13 +1196,13 @@ class PerceiverForImageClassificationLearned(PerceiverPreTrainedModel): pixel_values: Optional[torch.Tensor] = None, ) -> Union[Tuple, PerceiverClassifierOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1352,19 +1281,18 @@ class PerceiverForImageClassificationLearned(PerceiverPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Example use of Perceiver for image classification, for tasks such as ImageNet. + + This model uses fixed 2D Fourier position embeddings. As shown in the paper, this model can achieve a top-1 accuracy of + 79.0 on ImageNet, and 84.5 when pre-trained on a large-scale dataset (i.e. JFT). + + [`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] + (with `prep_type="pixels"`) to preprocess the input images, and + [`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of + [`PerceiverModel`] into classification logits. """ -Example use of Perceiver for image classification, for tasks such as ImageNet. - -This model uses fixed 2D Fourier position embeddings. As shown in the paper, this model can achieve a top-1 accuracy of -79.0 on ImageNet, and 84.5 when pre-trained on a large-scale dataset (i.e. JFT). - -[`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] -(with `prep_type="pixels"`) to preprocess the input images, and -[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of -[`PerceiverModel`] into classification logits. -""", - PERCEIVER_START_DOCSTRING, ) class PerceiverForImageClassificationFourier(PerceiverPreTrainedModel): def __init__(self, config): @@ -1398,8 +1326,7 @@ class PerceiverForImageClassificationFourier(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1412,13 +1339,13 @@ class PerceiverForImageClassificationFourier(PerceiverPreTrainedModel): pixel_values: Optional[torch.Tensor] = None, ) -> Union[Tuple, PerceiverClassifierOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1495,19 +1422,18 @@ class PerceiverForImageClassificationFourier(PerceiverPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Example use of Perceiver for image classification, for tasks such as ImageNet. + + This model uses a 2D conv+maxpool preprocessing network. As shown in the paper, this model can achieve a top-1 accuracy + of 82.1 on ImageNet. + + [`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] + (with `prep_type="conv"`) to preprocess the input images, and + [`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of + [`PerceiverModel`] into classification logits. """ -Example use of Perceiver for image classification, for tasks such as ImageNet. - -This model uses a 2D conv+maxpool preprocessing network. As shown in the paper, this model can achieve a top-1 accuracy -of 82.1 on ImageNet. - -[`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] -(with `prep_type="conv"`) to preprocess the input images, and -[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of -[`PerceiverModel`] into classification logits. -""", - PERCEIVER_START_DOCSTRING, ) class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel): def __init__(self, config): @@ -1542,8 +1468,7 @@ class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1556,13 +1481,13 @@ class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel): pixel_values: Optional[torch.Tensor] = None, ) -> Union[Tuple, PerceiverClassifierOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1639,19 +1564,18 @@ class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel): ) -@add_start_docstrings( - """ -Example use of Perceiver for optical flow, for tasks such as Sintel and KITTI. [`PerceiverForOpticalFlow`] uses -[`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with *prep_type="patches"*) to preprocess the -input images, and [`~models.perceiver.modeling_perceiver.PerceiverOpticalFlowDecoder`] to decode the latent -representation of [`PerceiverModel`]. +@auto_docstring( + custom_intro=""" + Example use of Perceiver for optical flow, for tasks such as Sintel and KITTI. [`PerceiverForOpticalFlow`] uses + [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with *prep_type="patches"*) to preprocess the + input images, and [`~models.perceiver.modeling_perceiver.PerceiverOpticalFlowDecoder`] to decode the latent + representation of [`PerceiverModel`]. -As input, one concatenates 2 subsequent frames along the channel dimension and extract a 3 x 3 patch around each pixel -(leading to 3 x 3 x 3 x 2 = 54 values for each pixel). Fixed Fourier position encodings are used to encode the position -of each pixel in the patch. Next, one applies the Perceiver encoder. To decode, one queries the latent representation -using the same encoding used for the input. -""", - PERCEIVER_START_DOCSTRING, + As input, one concatenates 2 subsequent frames along the channel dimension and extract a 3 x 3 patch around each pixel + (leading to 3 x 3 x 3 x 2 = 54 values for each pixel). Fixed Fourier position encodings are used to encode the position + of each pixel in the patch. Next, one applies the Perceiver encoder. To decode, one queries the latent representation + using the same encoding used for the input. + """ ) class PerceiverForOpticalFlow(PerceiverPreTrainedModel): def __init__(self, config): @@ -1703,8 +1627,7 @@ class PerceiverForOpticalFlow(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1716,11 +1639,11 @@ class PerceiverForOpticalFlow(PerceiverPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, PerceiverClassifierOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the optical flow loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: - Examples: ```python @@ -1768,35 +1691,34 @@ class PerceiverForOpticalFlow(PerceiverPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Example use of Perceiver for multimodal (video) autoencoding, for tasks such as Kinetics-700. + + [`PerceiverForMultimodalAutoencoding`] uses [`~models.perceiver.modeling_perceiver.PerceiverMultimodalPreprocessor`] to + preprocess the 3 modalities: images, audio and class labels. This preprocessor uses modality-specific preprocessors to + preprocess every modality separately, after which they are concatenated. Trainable position embeddings are used to pad + each modality to the same number of channels to make concatenation along the time dimension possible. Next, one applies + the Perceiver encoder. + + [`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] is used to decode the latent representation of + [`PerceiverModel`]. This decoder uses each modality-specific decoder to construct queries. The decoder queries are + created based on the inputs after preprocessing. However, autoencoding an entire video in a single forward pass is + computationally infeasible, hence one only uses parts of the decoder queries to do cross-attention with the latent + representation. This is determined by the subsampled indices for each modality, which can be provided as additional + input to the forward pass of [`PerceiverForMultimodalAutoencoding`]. + + [`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] also pads the decoder queries of the different + modalities to the same number of channels, in order to concatenate them along the time dimension. Next, cross-attention + is performed with the latent representation of [`PerceiverModel`]. + + Finally, [`~models.perceiver.modeling_perceiver.PerceiverMultiModalPostprocessor`] is used to turn this tensor into an + actual video. It first splits up the output into the different modalities, and then applies the respective + postprocessor for each modality. + + Note that, by masking the classification label during evaluation (i.e. simply providing a tensor of zeros for the + "label" modality), this auto-encoding model becomes a Kinetics 700 video classifier. """ -Example use of Perceiver for multimodal (video) autoencoding, for tasks such as Kinetics-700. - -[`PerceiverForMultimodalAutoencoding`] uses [`~models.perceiver.modeling_perceiver.PerceiverMultimodalPreprocessor`] to -preprocess the 3 modalities: images, audio and class labels. This preprocessor uses modality-specific preprocessors to -preprocess every modality separately, after which they are concatenated. Trainable position embeddings are used to pad -each modality to the same number of channels to make concatenation along the time dimension possible. Next, one applies -the Perceiver encoder. - -[`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] is used to decode the latent representation of -[`PerceiverModel`]. This decoder uses each modality-specific decoder to construct queries. The decoder queries are -created based on the inputs after preprocessing. However, autoencoding an entire video in a single forward pass is -computationally infeasible, hence one only uses parts of the decoder queries to do cross-attention with the latent -representation. This is determined by the subsampled indices for each modality, which can be provided as additional -input to the forward pass of [`PerceiverForMultimodalAutoencoding`]. - -[`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] also pads the decoder queries of the different -modalities to the same number of channels, in order to concatenate them along the time dimension. Next, cross-attention -is performed with the latent representation of [`PerceiverModel`]. - -Finally, [`~models.perceiver.modeling_perceiver.PerceiverMultiModalPostprocessor`] is used to turn this tensor into an -actual video. It first splits up the output into the different modalities, and then applies the respective -postprocessor for each modality. - -Note that, by masking the classification label during evaluation (i.e. simply providing a tensor of zeros for the -"label" modality), this auto-encoding model becomes a Kinetics 700 video classifier. -""", - PERCEIVER_START_DOCSTRING, ) class PerceiverForMultimodalAutoencoding(PerceiverPreTrainedModel): def __init__(self, config: PerceiverConfig): @@ -1914,8 +1836,7 @@ class PerceiverForMultimodalAutoencoding(PerceiverPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PERCEIVER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=PerceiverClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.Tensor] = None, @@ -1928,12 +1849,15 @@ class PerceiverForMultimodalAutoencoding(PerceiverPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, PerceiverClassifierOutput]: r""" + inputs (`torch.FloatTensor`): + Inputs to the perceiver. Can be anything: images, text, audio, video, etc. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - - Returns: + subsampled_output_points (`Dict[str, torch.Tensor]`, *optional*): + Dictionary of tensors used as queries for the decoder. The decoder maps these queries to the latent + representation of the model. Used for subsampled decoding, e.g. when only decoding certain image patches. Examples: diff --git a/src/transformers/models/persimmon/modeling_persimmon.py b/src/transformers/models/persimmon/modeling_persimmon.py index f273ce1705..2e53ce9bb4 100644 --- a/src/transformers/models/persimmon/modeling_persimmon.py +++ b/src/transformers/models/persimmon/modeling_persimmon.py @@ -38,15 +38,7 @@ from ...modeling_outputs import ( ) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_persimmon import PersimmonConfig @@ -58,9 +50,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "adept/persimmon-8b-base" -_CONFIG_FOR_DOC = "PersimmonConfig" - # Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Persimmon class PersimmonRotaryEmbedding(nn.Module): @@ -370,27 +359,7 @@ class PersimmonDecoderLayer(nn.Module): return outputs -PERSIMMON_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PersimmonConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Persimmon Model outputting raw hidden-states without any specific head on top.", - PERSIMMON_START_DOCSTRING, -) +@auto_docstring class PersimmonPreTrainedModel(PreTrainedModel): config_class = PersimmonConfig base_model_prefix = "model" @@ -416,85 +385,7 @@ class PersimmonPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -PERSIMMON_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Persimmon Model outputting raw hidden-states without any specific head on top.", - PERSIMMON_START_DOCSTRING, -) +@auto_docstring class PersimmonModel(PersimmonPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`PersimmonDecoderLayer`] @@ -527,7 +418,7 @@ class PersimmonModel(PersimmonPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PERSIMMON_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -812,8 +703,7 @@ class PersimmonForCausalLM(PersimmonPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(PERSIMMON_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -830,19 +720,10 @@ class PersimmonForCausalLM(PersimmonPreTrainedModel, GenerationMixin): **kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -902,8 +783,8 @@ class PersimmonForCausalLM(PersimmonPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Persimmon transformer with a sequence classification head on top (linear layer). [`PersimmonForSequenceClassification`] uses the last token in order to do the classification, as other causal @@ -914,8 +795,7 @@ class PersimmonForCausalLM(PersimmonPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - PERSIMMON_START_DOCSTRING, + """ ) # Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with LLAMA->PERSIMMON,Llama->Persimmon class PersimmonForSequenceClassification(PersimmonPreTrainedModel): @@ -935,7 +815,7 @@ class PersimmonForSequenceClassification(PersimmonPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PERSIMMON_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1004,13 +884,7 @@ class PersimmonForSequenceClassification(PersimmonPreTrainedModel): ) -@add_start_docstrings( - """ - The Persimmon Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - PERSIMMON_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.llama.modeling_llama.LlamaForTokenClassification with Llama->Persimmon, LLAMA->PERSIMMON class PersimmonForTokenClassification(PersimmonPreTrainedModel): def __init__(self, config): @@ -1036,12 +910,7 @@ class PersimmonForTokenClassification(PersimmonPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PERSIMMON_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/phi/modeling_phi.py b/src/transformers/models/phi/modeling_phi.py index f505bfd0e1..35df0f78cf 100644 --- a/src/transformers/models/phi/modeling_phi.py +++ b/src/transformers/models/phi/modeling_phi.py @@ -24,16 +24,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_phi import PhiConfig @@ -45,9 +36,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/phi-1" -_CONFIG_FOR_DOC = "PhiConfig" - def rotate_half(x): """Rotates half the hidden dims of the input.""" @@ -312,27 +300,7 @@ class PhiRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -PHI_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PhiConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Phi Model outputting raw hidden-states without any specific head on top.", - PHI_START_DOCSTRING, -) +@auto_docstring class PhiPreTrainedModel(PreTrainedModel): config_class = PhiConfig base_model_prefix = "model" @@ -362,88 +330,8 @@ class PhiPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -PHI_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Phi Model outputting raw hidden-states without any specific head on top.", - PHI_START_DOCSTRING, -) +@auto_docstring class PhiModel(PhiPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`PhiDecoderLayer`] - - Args: - config: PhiConfig - """ - def __init__(self, config: PhiConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -468,7 +356,7 @@ class PhiModel(PhiPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -700,6 +588,7 @@ class PhiModel(PhiPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class PhiForCausalLM(PhiPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -733,8 +622,7 @@ class PhiForCausalLM(PhiPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(PHI_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -751,19 +639,10 @@ class PhiForCausalLM(PhiPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -818,8 +697,8 @@ class PhiForCausalLM(PhiPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Phi Model transformer with a sequence classification head on top (linear layer). [`PhiForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -830,8 +709,7 @@ class PhiForCausalLM(PhiPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - PHI_START_DOCSTRING, + """ ) class PhiForSequenceClassification(PhiPreTrainedModel): def __init__(self, config): @@ -850,7 +728,7 @@ class PhiForSequenceClassification(PhiPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -919,13 +797,7 @@ class PhiForSequenceClassification(PhiPreTrainedModel): ) -@add_start_docstrings( - """ - The Phi Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - PHI_START_DOCSTRING, -) +@auto_docstring class PhiForTokenClassification(PhiPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -950,12 +822,7 @@ class PhiForTokenClassification(PhiPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/phi3/modeling_phi3.py b/src/transformers/models/phi3/modeling_phi3.py index 03422502e1..d0728df463 100644 --- a/src/transformers/models/phi3/modeling_phi3.py +++ b/src/transformers/models/phi3/modeling_phi3.py @@ -41,16 +41,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_phi3 import Phi3Config @@ -62,9 +53,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-mini-4k-instruct" -_CONFIG_FOR_DOC = "Phi3Config" - class Phi3MLP(nn.Module): def __init__(self, config): @@ -333,27 +321,7 @@ class Phi3DecoderLayer(GradientCheckpointingLayer): return outputs -PHI3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Phi3Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Phi3 Model outputting raw hidden-states without any specific head on top.", - PHI3_START_DOCSTRING, -) +@auto_docstring class Phi3PreTrainedModel(PreTrainedModel): config_class = Phi3Config base_model_prefix = "model" @@ -417,88 +385,8 @@ class Phi3RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -PHI3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Phi3 Model outputting raw hidden-states without any specific head on top.", - PHI3_START_DOCSTRING, -) +@auto_docstring class Phi3Model(Phi3PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`] - - Args: - config: Phi3Config - """ - def __init__(self, config: Phi3Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -522,7 +410,7 @@ class Phi3Model(Phi3PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -775,6 +663,7 @@ class Phi3Model(Phi3PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -808,8 +697,7 @@ class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -826,19 +714,10 @@ class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -932,8 +811,8 @@ class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin): return model_inputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Phi3 Model transformer with a sequence classification head on top (linear layer). [`Phi3ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -944,8 +823,7 @@ class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - PHI3_START_DOCSTRING, + """ ) class Phi3ForSequenceClassification(Phi3PreTrainedModel): def __init__(self, config): @@ -964,7 +842,7 @@ class Phi3ForSequenceClassification(Phi3PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1033,13 +911,7 @@ class Phi3ForSequenceClassification(Phi3PreTrainedModel): ) -@add_start_docstrings( - """ - The Phi3 Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - PHI3_START_DOCSTRING, -) +@auto_docstring class Phi3ForTokenClassification(Phi3PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1064,12 +936,7 @@ class Phi3ForTokenClassification(Phi3PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py index f2f691139d..a373e72367 100644 --- a/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py +++ b/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py @@ -46,15 +46,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging, torch_int from .configuration_phi4_multimodal import Phi4MultimodalAudioConfig, Phi4MultimodalConfig, Phi4MultimodalVisionConfig @@ -380,12 +372,8 @@ def default_flax_embed_init(tensor): variance_scaling_(tensor, mode="fan_in", distribution="normal") +@auto_docstring class Phi4MultimodalVisionPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Phi4MultimodalVisionConfig base_model_prefix = "phi4_vision" supports_gradient_checkpointing = True @@ -1010,6 +998,7 @@ class Phi4MultimodalAudioMeanVarianceNormLayer(nn.Module): return (x - self.global_mean) * self.global_invstd +@auto_docstring class Phi4MultimodalAudioPreTrainedModel(PreTrainedModel): config_class = Phi4MultimodalAudioConfig supports_gradient_checkpointing = True @@ -1638,27 +1627,7 @@ class Phi4MultimodalRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -PHI4_MULTIMODAL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Phi4MultimodalConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Phi4Multimodal Model outputting raw hidden-states without any specific head on top.", - PHI4_MULTIMODAL_START_DOCSTRING, -) +@auto_docstring class Phi4MultimodalPreTrainedModel(PreTrainedModel): config_class = Phi4MultimodalConfig base_model_prefix = "model" @@ -1691,82 +1660,8 @@ class Phi4MultimodalPreTrainedModel(PreTrainedModel): module.sub_img_feature_extensor.data.zero_() -PHI4_MULTIMODAL_MODEL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`)`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - See our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - image_pixel_values (`torch.FloatTensor`, *optional*): - If the input contains images, these correspond to the pixel values after transformations (as returned by - the Processor) - image_sizes (`torch.LongTensor`, *optional*): - If the input contains images, these correspond to size of each image. - image_attention_mask (`torch.LongTensor`, *optional*): - Attention mask for the images. - audio_input_features (`torch.FloatTensor`, *optional*): - If the input contains audio samples, these correspond to the values after transformation (as returned by - the Processor). - audio_embed_sizes (`torch.Tensor`, *optional*): - Size of the audio inputs. - audio_attention_mask (`torch.Tensor, *optional*): - Attention mask for the audio inputs. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Phi4Multimodal Model outputting raw hidden-states without any specific head on top.", - PHI4_MULTIMODAL_START_DOCSTRING, -) +@auto_docstring class Phi4MultimodalModel(Phi4MultimodalPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi4MultimodalMMDecoderLayer`] - Args: - config: Phi4MultimodalMMConfig - """ - def __init__(self, config: Phi4MultimodalConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1795,7 +1690,7 @@ class Phi4MultimodalModel(Phi4MultimodalPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHI4_MULTIMODAL_MODEL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1815,6 +1710,22 @@ class Phi4MultimodalModel(Phi4MultimodalPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs, ) -> BaseModelOutputWithPast: + r""" + image_pixel_values (`torch.FloatTensor`, *optional*): + If the input contains images, these correspond to the pixel values after transformations (as returned by + the Processor) + image_sizes (`torch.LongTensor`, *optional*): + If the input contains images, these correspond to size of each image. + image_attention_mask (`torch.LongTensor`, *optional*): + Attention mask for the images. + audio_input_features (`torch.FloatTensor`, *optional*): + If the input contains audio samples, these correspond to the values after transformation (as returned by + the Processor). + audio_embed_sizes (`torch.Tensor`, *optional*): + Size of the audio inputs. + audio_attention_mask (`torch.Tensor, *optional*): + Attention mask for the audio inputs. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -2057,6 +1968,7 @@ class Phi4MultimodalModel(Phi4MultimodalPreTrainedModel): return causal_mask +@auto_docstring class Phi4MultimodalForCausalLM(Phi4MultimodalPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -2090,8 +2002,7 @@ class Phi4MultimodalForCausalLM(Phi4MultimodalPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(PHI4_MULTIMODAL_MODEL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=Phi4MultimodalConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2114,18 +2025,24 @@ class Phi4MultimodalForCausalLM(Phi4MultimodalPreTrainedModel, GenerationMixin): **kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - Returns: + image_pixel_values (`torch.FloatTensor`, *optional*): + If the input contains images, these correspond to the pixel values after transformations (as returned by + the Processor) + image_sizes (`torch.LongTensor`, *optional*): + If the input contains images, these correspond to size of each image. + image_attention_mask (`torch.LongTensor`, *optional*): + Attention mask for the images. + audio_input_features (`torch.FloatTensor`, *optional*): + If the input contains audio samples, these correspond to the values after transformation (as returned by + the Processor). + audio_embed_sizes (`torch.Tensor`, *optional*): + Size of the audio inputs. + audio_attention_mask (`torch.Tensor, *optional*): + Attention mask for the audio inputs. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python diff --git a/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py index d13fa8dc8b..5b3a585bc1 100644 --- a/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py +++ b/src/transformers/models/phi4_multimodal/modular_phi4_multimodal.py @@ -33,12 +33,7 @@ from ...modeling_outputs import ( CausalLMOutputWithPast, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..phi3.configuration_phi3 import Phi3Config from ..phi3.modeling_phi3 import ( Phi3DecoderLayer, @@ -1119,6 +1114,7 @@ class Phi4MultimodalAudioMeanVarianceNormLayer(nn.Module): return (x - self.global_mean) * self.global_invstd +@auto_docstring class Phi4MultimodalAudioPreTrainedModel(PreTrainedModel): config_class = Phi4MultimodalAudioConfig supports_gradient_checkpointing = True @@ -1457,71 +1453,6 @@ class Phi4MultimodalFeatureEmbedding(nn.Module): return inputs_embeds -PHI4_MULTIMODAL_MODEL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`)`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - See our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - image_pixel_values (`torch.FloatTensor`, *optional*): - If the input contains images, these correspond to the pixel values after transformations (as returned by - the Processor) - image_sizes (`torch.LongTensor`, *optional*): - If the input contains images, these correspond to size of each image. - image_attention_mask (`torch.LongTensor`, *optional*): - Attention mask for the images. - audio_input_features (`torch.FloatTensor`, *optional*): - If the input contains audio samples, these correspond to the values after transformation (as returned by - the Processor). - audio_embed_sizes (`torch.Tensor`, *optional*): - Size of the audio inputs. - audio_attention_mask (`torch.Tensor, *optional*): - Attention mask for the audio inputs. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - class Phi4MultimodalRotaryEmbedding(Phi3RotaryEmbedding): pass @@ -1545,12 +1476,6 @@ class Phi4MultimodalPreTrainedModel(Phi3PreTrainedModel): class Phi4MultimodalModel(Phi3Model, nn.Module): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi4MultimodalMMDecoderLayer`] - Args: - config: Phi4MultimodalMMConfig - """ - def __init__(self, config: Phi4MultimodalConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -1570,8 +1495,6 @@ class Phi4MultimodalModel(Phi3Model, nn.Module): # Initialize weights and apply final processing self.post_init() - @can_return_tuple - @add_start_docstrings_to_model_forward(PHI4_MULTIMODAL_MODEL_INPUTS_DOCSTRING) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1591,6 +1514,22 @@ class Phi4MultimodalModel(Phi3Model, nn.Module): cache_position: Optional[torch.LongTensor] = None, **kwargs, ) -> BaseModelOutputWithPast: + r""" + image_pixel_values (`torch.FloatTensor`, *optional*): + If the input contains images, these correspond to the pixel values after transformations (as returned by + the Processor) + image_sizes (`torch.LongTensor`, *optional*): + If the input contains images, these correspond to size of each image. + image_attention_mask (`torch.LongTensor`, *optional*): + Attention mask for the images. + audio_input_features (`torch.FloatTensor`, *optional*): + If the input contains audio samples, these correspond to the values after transformation (as returned by + the Processor). + audio_embed_sizes (`torch.Tensor`, *optional*): + Size of the audio inputs. + audio_attention_mask (`torch.Tensor, *optional*): + Attention mask for the audio inputs. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1691,9 +1630,6 @@ class Phi4MultimodalForCausalLM(Phi3ForCausalLM, nn.Module): # Initialize weights and apply final processing self.post_init() - @can_return_tuple - @add_start_docstrings_to_model_forward(PHI4_MULTIMODAL_MODEL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=Phi4MultimodalConfig) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1716,18 +1652,24 @@ class Phi4MultimodalForCausalLM(Phi3ForCausalLM, nn.Module): **kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - Returns: + image_pixel_values (`torch.FloatTensor`, *optional*): + If the input contains images, these correspond to the pixel values after transformations (as returned by + the Processor) + image_sizes (`torch.LongTensor`, *optional*): + If the input contains images, these correspond to size of each image. + image_attention_mask (`torch.LongTensor`, *optional*): + Attention mask for the images. + audio_input_features (`torch.FloatTensor`, *optional*): + If the input contains audio samples, these correspond to the values after transformation (as returned by + the Processor). + audio_embed_sizes (`torch.Tensor`, *optional*): + Size of the audio inputs. + audio_attention_mask (`torch.Tensor, *optional*): + Attention mask for the audio inputs. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python diff --git a/src/transformers/models/phimoe/modeling_phimoe.py b/src/transformers/models/phimoe/modeling_phimoe.py index 388505cf33..35c24718fb 100644 --- a/src/transformers/models/phimoe/modeling_phimoe.py +++ b/src/transformers/models/phimoe/modeling_phimoe.py @@ -27,21 +27,10 @@ from ...cache_utils import Cache, DynamicCache, SlidingWindowCache, StaticCache from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter, _prepare_4d_causal_attention_mask from ...modeling_flash_attention_utils import is_flash_attn_available -from ...modeling_outputs import ( - MoeCausalLMOutputWithPast, - MoeModelOutputWithPast, - SequenceClassifierOutputWithPast, -) +from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast, SequenceClassifierOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_phimoe import PhimoeConfig @@ -61,8 +50,6 @@ _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_m logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "PhimoeConfig" - # Copied from transformers.models.mixtral.modeling_mixtral.load_balancing_loss_func def load_balancing_loss_func( @@ -888,25 +875,7 @@ class PhimoeDecoderLayer(nn.Module): return outputs -PHIMOE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - Parameters: - config ([`PhimoeConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Phimoe Model outputting raw hidden-states without any specific head on top.", - PHIMOE_START_DOCSTRING, -) +@auto_docstring class PhimoePreTrainedModel(PreTrainedModel): config_class = PhimoeConfig base_model_prefix = "model" @@ -934,81 +903,7 @@ class PhimoePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -PHIMOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Phimoe Model outputting raw hidden-states without any specific head on top.", - PHIMOE_START_DOCSTRING, -) +@auto_docstring class PhimoeModel(PhimoePreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`PhimoeDecoderLayer`] @@ -1040,7 +935,7 @@ class PhimoeModel(PhimoePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHIMOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1369,9 +1264,7 @@ class PhimoeForCausalLM(PhimoePreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(PHIMOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - # Ignore copy + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1389,18 +1282,11 @@ class PhimoeForCausalLM(PhimoePreTrainedModel, GenerationMixin): **loss_kwargs, ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - Returns: Example: ```python >>> from transformers import AutoTokenizer, PhimoeForCausalLM @@ -1516,8 +1402,8 @@ class PhimoeForCausalLM(PhimoePreTrainedModel, GenerationMixin): return model_inputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Phimoe Model transformer with a sequence classification head on top (linear layer). [`PhimoeForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. @@ -1526,10 +1412,8 @@ class PhimoeForCausalLM(PhimoePreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - PHIMOE_START_DOCSTRING, + """ ) - # Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phimoe, LLAMA->PHIMOE, BaseModelOutputWithPast->MoeModelOutputWithPast class PhimoeForSequenceClassification(PhimoePreTrainedModel): def __init__(self, config): @@ -1548,7 +1432,7 @@ class PhimoeForSequenceClassification(PhimoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(PHIMOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/pix2struct/modeling_pix2struct.py b/src/transformers/models/pix2struct/modeling_pix2struct.py index ac5c8b4877..243c81a0c8 100644 --- a/src/transformers/models/pix2struct/modeling_pix2struct.py +++ b/src/transformers/models/pix2struct/modeling_pix2struct.py @@ -37,13 +37,11 @@ from ...pytorch_utils import ALL_LAYERNORM_LAYERS from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from .configuration_pix2struct import Pix2StructConfig, Pix2StructTextConfig, Pix2StructVisionConfig @@ -57,7 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) # General docstring -_CONFIG_FOR_DOC = "Pix2StructConfig" # Adapted from transformers.models.t5.modeling_t5.T5LayerNorm with T5->Pix2Struct @@ -361,12 +358,8 @@ class Pix2StructVisionEncoder(nn.Module): ) +@auto_docstring class Pix2StructPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Pix2StructConfig _supports_cache_class = True _supports_static_cache = False @@ -490,48 +483,7 @@ class Pix2StructPreTrainedModel(PreTrainedModel): return shifted_input_ids -PIX2STRUCT_VISION_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Pix2StructConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PIX2STRUCT_VISION_INPUTS_DOCSTRING = r""" - Args: - flattened_patches (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels x patch_height x patch_width)`): - Flattened and padded pixel values. These values can be obtained using [`AutoImageProcessor`]. See - [`Pix2StructVisionImageProcessor.__call__`] for details. Check the [original - paper](https://arxiv.org/abs/2210.03347) (figure 5) for more details. - - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Pix2StructVision Model transformer outputting raw hidden-states without any specific head on top.", - PIX2STRUCT_VISION_START_DOCSTRING, -) +@auto_docstring class Pix2StructVisionModel(Pix2StructPreTrainedModel): config_class = Pix2StructVisionConfig main_input_name = "flattened_patches" @@ -561,8 +513,7 @@ class Pix2StructVisionModel(Pix2StructPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(PIX2STRUCT_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, flattened_patches: Optional[torch.Tensor] = None, @@ -573,7 +524,10 @@ class Pix2StructVisionModel(Pix2StructPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + flattened_patches (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels x patch_height x patch_width)`): + Flattened and padded pixel values. These values can be obtained using [`AutoImageProcessor`]. See + [`Pix2StructVisionImageProcessor.__call__`] for details. Check the [original + paper](https://arxiv.org/abs/2210.03347) (figure 5) for more details. Example: @@ -1066,218 +1020,10 @@ class Pix2StructTextBlock(nn.Module): return outputs -PIX2STRUCT_START_DOCSTRING = r""" - - The Pix2Struct model was proposed in [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language - Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, - Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. It's an encoder decoder - transformer pre-trained in a image-to-text setting. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config (Union[`Pix2StructConfig`, `Pix2StructTextConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PIX2STRUCT_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Pix2StructText is a model with relative position - embeddings so you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - [What are input IDs?](../glossary#input-ids) - - To know more on how to prepare `input_ids` for pretraining take a look a [Pix2StructText - Training](./t5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Pix2StructText uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [Pix2StructText - Training](./t5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention layers. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - -PIX2STRUCT_INPUTS_DOCSTRING = r""" - Args: - flattened_patches (`torch.FloatTensor` of shape `(batch_size, seq_length, hidden_size)`): - Flattened pixel patches. the `hidden_size` is obtained by the following formula: `hidden_size` = - `num_channels` * `patch_size` * `patch_size` - - The process of flattening the pixel patches is done by `Pix2StructProcessor`. - - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Pix2StructText uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [Pix2StructText - Training](./t5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention layers. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss for the decoder. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The standalone text decoder of Pix2Struct", - PIX2STRUCT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The standalone text decoder of Pix2Struct + """ ) class Pix2StructTextModel(Pix2StructPreTrainedModel): config_class = Pix2StructTextConfig @@ -1347,8 +1093,7 @@ class Pix2StructTextModel(Pix2StructPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(PIX2STRUCT_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1368,7 +1113,23 @@ class Pix2StructTextModel(Pix2StructPreTrainedModel): **kwargs, ) -> Union[Tuple[torch.FloatTensor, ...], CausalLMOutputWithCrossAttentions]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Pix2StructText is a model with relative position + embeddings so you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [Pix2StructText + Training](./t5#training). + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1710,9 +1471,10 @@ class Pix2StructTextModel(Pix2StructPreTrainedModel): return causal_mask -@add_start_docstrings( - "A conditional generation model with a language modeling head. Can be used for sequence generation tasks.", - PIX2STRUCT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + A conditional generation model with a language modeling head. Can be used for sequence generation tasks. + """ ) class Pix2StructForConditionalGeneration(Pix2StructPreTrainedModel, GenerationMixin): config_class = Pix2StructConfig @@ -1748,8 +1510,7 @@ class Pix2StructForConditionalGeneration(Pix2StructPreTrainedModel, GenerationMi def get_encoder(self): return self.encoder - @add_start_docstrings_to_model_forward(PIX2STRUCT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, flattened_patches: Optional[torch.FloatTensor] = None, @@ -1770,7 +1531,42 @@ class Pix2StructForConditionalGeneration(Pix2StructPreTrainedModel, GenerationMi cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + flattened_patches (`torch.FloatTensor` of shape `(batch_size, seq_length, hidden_size)`): + Flattened pixel patches. the `hidden_size` is obtained by the following formula: `hidden_size` = + `num_channels` * `patch_size` * `patch_size` + + The process of flattening the pixel patches is done by `Pix2StructProcessor`. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Pix2StructText uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [Pix2StructText + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss for the decoder. Example: diff --git a/src/transformers/models/pixtral/image_processing_pixtral_fast.py b/src/transformers/models/pixtral/image_processing_pixtral_fast.py index 0cb4673038..0b5bd193c0 100644 --- a/src/transformers/models/pixtral/image_processing_pixtral_fast.py +++ b/src/transformers/models/pixtral/image_processing_pixtral_fast.py @@ -18,8 +18,6 @@ from typing import Dict, List, Optional, Union from ...image_processing_utils import BatchFeature, get_size_dict from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -33,7 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -61,17 +59,15 @@ if is_torchvision_available(): class PixtralFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + patch_size (`Dict[str, int]` *optional*, defaults to `{"height": 16, "width": 16}`): + Size of the patches in the model, used to calculate the output image size. Can be overridden by `patch_size` in the `preprocess` method. + """ + patch_size: Optional[Dict[str, int]] -@add_start_docstrings( - r"Constructs a fast ConvNeXT image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - patch_size (`Dict[str, int]` *optional*, defaults to `{"height": 16, "width": 16}`): - Size of the patches in the model, used to calculate the output image size. Can be overridden by `patch_size` in the `preprocess` method. - """, -) +@auto_docstring class PixtralImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = [0.48145466, 0.4578275, 0.40821073] @@ -88,13 +84,7 @@ class PixtralImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[PixtralFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - patch_size (`Dict[str, int]` *optional*, defaults to `{"height": 16, "width": 16}`): - Size of the patches in the model, used to calculate the output image size. Can be overridden by `patch_size` in the `preprocess` method. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[PixtralFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/pixtral/modeling_pixtral.py b/src/transformers/models/pixtral/modeling_pixtral.py index 171bf63986..61d5ada712 100644 --- a/src/transformers/models/pixtral/modeling_pixtral.py +++ b/src/transformers/models/pixtral/modeling_pixtral.py @@ -27,12 +27,7 @@ from ...modeling_outputs import BaseModelOutput from ...modeling_rope_utils import dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, -) +from ...utils import auto_docstring, can_return_tuple, logging from .configuration_pixtral import PixtralVisionConfig @@ -407,23 +402,7 @@ class PixtralTransformer(nn.Module): ) -PIXTRAL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PixtralVisionConfig`]): - Model configuration class with all the parameters of the vision encoder. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - +@auto_docstring class PixtralPreTrainedModel(PreTrainedModel): config_class = PixtralVisionConfig base_model_prefix = "model" @@ -449,24 +428,6 @@ class PixtralPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -PIXTRAL_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`AutoImageProcessor.__call__`] - for details. - image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*): - The sizes of the images in the batch, being (height, width) for each image. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - def generate_block_attention_mask(patch_embeds_list, tensor): dtype = tensor.dtype device = tensor.device @@ -483,10 +444,7 @@ def generate_block_attention_mask(patch_embeds_list, tensor): return causal_mask -@add_start_docstrings( - "The bare Pixtral vision encoder outputting raw hidden-states without any specific head on top.", - PIXTRAL_START_DOCSTRING, -) +@auto_docstring class PixtralVisionModel(PixtralPreTrainedModel): base_model_prefix = "vision_encoder" @@ -511,7 +469,7 @@ class PixtralVisionModel(PixtralPreTrainedModel): return self.patch_conv @can_return_tuple - @add_start_docstrings_to_model_forward(PIXTRAL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -522,11 +480,6 @@ class PixtralVisionModel(PixtralPreTrainedModel): *args, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, BaseModelOutput]: - """ - Returns: - pixel_values: tensor of token features for - all tokens of all images of shape (N_toks, D) - """ if image_sizes is None: batch_size, _, height, width = pixel_values.shape image_sizes = [(height, width)] * batch_size diff --git a/src/transformers/models/plbart/modeling_plbart.py b/src/transformers/models/plbart/modeling_plbart.py index e625c20251..22157b29c7 100644 --- a/src/transformers/models/plbart/modeling_plbart.py +++ b/src/transformers/models/plbart/modeling_plbart.py @@ -40,22 +40,12 @@ from ...modeling_outputs import ( Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_plbart import PLBartConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "uclanlp/plbart-base" -_CONFIG_FOR_DOC = "PLBartConfig" - # Copied from transformers.models.mbart.modeling_mbart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int): @@ -495,6 +485,7 @@ class PLBartClassificationHead(nn.Module): return hidden_states +@auto_docstring class PLBartPreTrainedModel(PreTrainedModel): config_class = PLBartConfig base_model_prefix = "model" @@ -513,144 +504,6 @@ class PLBartPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -PLBART_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`PLBartConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PLBART_GENERATION_EXAMPLE = r""" - Mask-filling example: - - ```python - >>> from transformers import AutoTokenizer, PLBartForConditionalGeneration - - >>> model = PLBartForConditionalGeneration.from_pretrained("uclanlp/plbart-base") - >>> tokenizer = AutoTokenizer.from_pretrained("uclanlp/plbart-base") - - >>> # en_XX is the language symbol id for English - >>> TXT = " Is 0 the Fibonacci number ? en_XX" - >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="pt").input_ids - - >>> logits = model(input_ids).logits - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() - >>> probs = logits[0, masked_index].softmax(dim=0) - >>> values, predictions = probs.topk(5) - - >>> tokenizer.decode(predictions).split() - ['first', 'same', 'highest', 'result', 'number'] - ``` -""" - -PLBART_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. - See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. - See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that - varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (: - obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: - generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (: - obj:*torch.Tensor* of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify - selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (: - obj:*tuple(tuple(torch.FloatTensor))*, *optional*, returned when `use_cache=True` is passed or when - `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple - having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional - tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (: - obj:*torch.FloatTensor* of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, - instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful - if you want more control over how to convert `input_ids` indices into associated vectors than the model's - internal embedding lookup matrix. - decoder_inputs_embeds (: - obj:*torch.FloatTensor* of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.bart.modeling_bart.BartEncoder with Bart->PLBart class PLBartEncoder(PLBartPreTrainedModel): """ @@ -1128,10 +981,7 @@ class PLBartDecoder(PLBartPreTrainedModel): ) -@add_start_docstrings( - "The bare PLBART Model outputting raw hidden-states without any specific head on top.", - PLBART_START_DOCSTRING, -) +@auto_docstring class PLBartModel(PLBartPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1166,12 +1016,7 @@ class PLBartModel(PLBartPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(PLBART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1190,6 +1035,35 @@ class PLBartModel(PLBartPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqModelOutput]: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. + See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that + varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (: + obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: + generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. + cross_attn_head_mask (: + obj:*torch.Tensor* of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify + selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1251,9 +1125,10 @@ class PLBartModel(PLBartPreTrainedModel): ) -@add_start_docstrings( - "The PLBART Model with a language modeling head. Can be used for code-to-text, text-to-code and code-to-code.", - PLBART_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The PLBART Model with a language modeling head. Can be used for code-to-text, text-to-code and code-to-code. + """ ) class PLBartForConditionalGeneration(PLBartPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1296,9 +1171,7 @@ class PLBartForConditionalGeneration(PLBartPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(PLBART_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(PLBART_GENERATION_EXAMPLE) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1319,13 +1192,58 @@ class PLBartForConditionalGeneration(PLBartPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. + See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that + varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (: + obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: + generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. + cross_attn_head_mask (: + obj:*torch.Tensor* of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify + selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + Example Mask-filling: + ```python + >>> from transformers import AutoTokenizer, PLBartForConditionalGeneration + + >>> model = PLBartForConditionalGeneration.from_pretrained("uclanlp/plbart-base") + >>> tokenizer = AutoTokenizer.from_pretrained("uclanlp/plbart-base") + + >>> # en_XX is the language symbol id for English + >>> TXT = " Is 0 the Fibonacci number ? en_XX" + >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="pt").input_ids + + >>> logits = model(input_ids).logits + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = logits[0, masked_index].softmax(dim=0) + >>> values, predictions = probs.topk(5) + + >>> tokenizer.decode(predictions).split() + ['first', 'same', 'highest', 'result', 'number'] + ``` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1389,12 +1307,11 @@ class PLBartForConditionalGeneration(PLBartPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" PLBart model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for code classification. - """, - PLBART_START_DOCSTRING, + """ ) class PLBartForSequenceClassification(PLBartPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1412,13 +1329,8 @@ class PLBartForSequenceClassification(PLBartPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PLBART_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) - # Copied from transformers.models.bart.modeling_bart.BartForSequenceClassification.forward + @auto_docstring + # Ignore copy def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1438,6 +1350,33 @@ class PLBartForSequenceClassification(PLBartPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint. + See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that + varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (: + obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: + generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. + cross_attn_head_mask (: + obj:*torch.Tensor* of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify + selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). @@ -1567,7 +1506,7 @@ class PLBartForCausalLM(PLBartPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1585,72 +1524,15 @@ class PLBartForCausalLM(PLBartPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/poolformer/image_processing_poolformer_fast.py b/src/transformers/models/poolformer/image_processing_poolformer_fast.py index 37e81a5f5f..0dcbfd15c5 100644 --- a/src/transformers/models/poolformer/image_processing_poolformer_fast.py +++ b/src/transformers/models/poolformer/image_processing_poolformer_fast.py @@ -17,8 +17,6 @@ from typing import Optional, Union from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, BatchFeature, DefaultFastImageProcessorKwargs, @@ -41,7 +39,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -59,17 +57,16 @@ if is_torchvision_available(): class PoolFormerFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + Args: + crop_pct (`float`, *optional*, defaults to `self.crop_pct`): + Percentage of the image to crop. Only has an effect if `do_resize` is set to `True`. + """ + crop_pct: Optional[float] -@add_start_docstrings( - "Constructs a fast PoolFormer image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - crop_pct (`float`, *optional*, defaults to `self.crop_pct`): - Percentage of the image to crop. Only has an effect if `do_resize` is set to `True`. - """, -) +@auto_docstring class PoolFormerImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = IMAGENET_DEFAULT_MEAN @@ -87,13 +84,7 @@ class PoolFormerImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[PoolFormerFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - crop_pct (`float`, *optional*, defaults to `self.crop_pct`): - Percentage of the image to crop. Only has an effect if `do_resize` is set to `True`. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[PoolFormerFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/poolformer/modeling_poolformer.py b/src/transformers/models/poolformer/modeling_poolformer.py index 5b2ed8868b..9fa2ff3743 100755 --- a/src/transformers/models/poolformer/modeling_poolformer.py +++ b/src/transformers/models/poolformer/modeling_poolformer.py @@ -25,23 +25,12 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_poolformer import PoolFormerConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "PoolFormerConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "sail/poolformer_s12" -_EXPECTED_OUTPUT_SHAPE = [1, 512, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "sail/poolformer_s12" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: @@ -255,12 +244,8 @@ class PoolFormerEncoder(nn.Module): return BaseModelOutputWithNoAttention(last_hidden_state=hidden_states, hidden_states=all_hidden_states) +@auto_docstring class PoolFormerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = PoolFormerConfig base_model_prefix = "poolformer" main_input_name = "pixel_values" @@ -281,29 +266,7 @@ class PoolFormerPreTrainedModel(PreTrainedModel): module.layer_scale_2.data.fill_(self.config.layer_scale_init_value) -POOLFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -POOLFORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`PoolFormerImageProcessor.__call__`] for details. -""" - - -@add_start_docstrings( - "The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top.", - POOLFORMER_START_DOCSTRING, -) +@auto_docstring class PoolFormerModel(PoolFormerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -317,14 +280,7 @@ class PoolFormerModel(PoolFormerPreTrainedModel): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(POOLFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -365,11 +321,10 @@ class PoolFormerFinalPooler(nn.Module): return output -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" PoolFormer Model transformer with an image classification head on top - """, - POOLFORMER_START_DOCSTRING, + """ ) class PoolFormerForImageClassification(PoolFormerPreTrainedModel): def __init__(self, config): @@ -387,13 +342,7 @@ class PoolFormerForImageClassification(PoolFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(POOLFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, diff --git a/src/transformers/models/pop2piano/modeling_pop2piano.py b/src/transformers/models/pop2piano/modeling_pop2piano.py index a99f52747f..5191a6a490 100644 --- a/src/transformers/models/pop2piano/modeling_pop2piano.py +++ b/src/transformers/models/pop2piano/modeling_pop2piano.py @@ -28,22 +28,10 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache from ...generation import GenerationMixin from ...modeling_attn_mask_utils import AttentionMaskConverter -from ...modeling_outputs import ( - BaseModelOutput, - BaseModelOutputWithPastAndCrossAttentions, - Seq2SeqLMOutput, -) +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, Seq2SeqLMOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import ALL_LAYERNORM_LAYERS, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - is_torch_fx_proxy, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging from .configuration_pop2piano import Pop2PianoConfig @@ -71,87 +59,6 @@ except Exception: pass -_CONFIG_FOR_DOC = "Pop2PianoConfig" -_CHECKPOINT_FOR_DOC = "sweetcocoa/pop2piano" - - -POP2PIANO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Pop2Piano is a model with relative position embeddings - so you should be able to pad the inputs on both the right and the left. Indices can be obtained using - [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. - [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining - take a look a [Pop2Piano Training](./Pop2Piano#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using - [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - [What are decoder input IDs?](../glossary#decoder-input-ids) Pop2Piano uses the `pad_token_id` as the - starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last - `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Does the same task as `inputs_embeds`. If `inputs_embeds` is not present but `input_features` is present - then `input_features` will be considered as `inputs_embeds`. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If - `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value of - `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - - # Copied from transformers.models.t5.modeling_t5.T5LayerNorm with T5->Pop2Piano class Pop2PianoLayerNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -664,12 +571,8 @@ class Pop2PianoBlock(nn.Module): return outputs # hidden-states, past_key_value, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights) +@auto_docstring class Pop2PianoPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Pop2PianoConfig base_model_prefix = "transformer" is_parallelizable = False @@ -1137,23 +1040,11 @@ class Pop2PianoConcatEmbeddingToMel(nn.Module): return inputs_embeds -Pop2Piano_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Pop2PianoConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings("""Pop2Piano Model with a `language modeling` head on top.""", Pop2Piano_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + Pop2Piano Model with a `language modeling` head on top. + """ +) class Pop2PianoForConditionalGeneration(Pop2PianoPreTrainedModel, GenerationMixin): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"] @@ -1253,8 +1144,7 @@ class Pop2PianoForConditionalGeneration(Pop2PianoPreTrainedModel, GenerationMixi return input_features, None - @add_start_docstrings_to_model_forward(POP2PIANO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1277,11 +1167,38 @@ class Pop2PianoForConditionalGeneration(Pop2PianoPreTrainedModel, GenerationMixi cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Pop2Piano is a model with relative position embeddings + so you should be able to pad the inputs on both the right and the left. Indices can be obtained using + [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. + [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining + take a look a [Pop2Piano Training](./Pop2Piano#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using + [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + [What are decoder input IDs?](../glossary#decoder-input-ids) Pop2Piano uses the `pad_token_id` as the + starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last + `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Does the same task as `inputs_embeds`. If `inputs_embeds` is not present but `input_features` is present + then `input_features` will be considered as `inputs_embeds`. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: """ use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/prompt_depth_anything/modeling_prompt_depth_anything.py b/src/transformers/models/prompt_depth_anything/modeling_prompt_depth_anything.py index 7a1e2c7e47..f4336cc1e3 100644 --- a/src/transformers/models/prompt_depth_anything/modeling_prompt_depth_anything.py +++ b/src/transformers/models/prompt_depth_anything/modeling_prompt_depth_anything.py @@ -23,16 +23,13 @@ import torch.nn as nn from transformers.utils.generic import torch_int -from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings from ...modeling_outputs import DepthEstimatorOutput from ...modeling_utils import PreTrainedModel +from ...utils import auto_docstring from ...utils.backbone_utils import load_backbone from .configuration_prompt_depth_anything import PromptDepthAnythingConfig -_CONFIG_FOR_DOC = "PromptDepthAnythingConfig" - - class PromptDepthAnythingLayer(nn.Module): def __init__(self, config: PromptDepthAnythingConfig): super().__init__() @@ -241,12 +238,8 @@ class PromptDepthAnythingDepthEstimationHead(nn.Module): return predicted_depth +@auto_docstring class PromptDepthAnythingPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = PromptDepthAnythingConfig base_model_prefix = "prompt_depth_anything" main_input_name = "pixel_values" @@ -379,44 +372,10 @@ class PromptDepthAnythingNeck(nn.Module): return output -PROMPT_DEPTH_ANYTHING_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`PromptDepthAnythingConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PROMPT_DEPTH_ANYTHING_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] - for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - prompt_depth (`torch.FloatTensor` of shape `(batch_size, 1, height, width)`, *optional*): - Prompt depth is the sparse or low-resolution depth obtained from multi-view geometry or a - low-resolution depth sensor. It generally has shape (height, width), where height - and width can be smaller than those of the images. It is optional and can be None, which means no prompt depth - will be used. If it is None, the output will be a monocular relative depth. - The values are recommended to be in meters, but this is not necessary. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Prompt Depth Anything Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. - """, - PROMPT_DEPTH_ANYTHING_START_DOCSTRING, + """ ) class PromptDepthAnythingForDepthEstimation(PromptDepthAnythingPreTrainedModel): _no_split_modules = ["DPTViTEmbeddings"] @@ -431,8 +390,7 @@ class PromptDepthAnythingForDepthEstimation(PromptDepthAnythingPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PROMPT_DEPTH_ANYTHING_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -443,12 +401,14 @@ class PromptDepthAnythingForDepthEstimation(PromptDepthAnythingPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], DepthEstimatorOutput]: r""" - labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Ground truth depth estimation maps for computing the loss. + prompt_depth (`torch.FloatTensor` of shape `(batch_size, 1, height, width)`, *optional*): + Prompt depth is the sparse or low-resolution depth obtained from multi-view geometry or a + low-resolution depth sensor. It generally has shape (height, width), where height + and width can be smaller than those of the images. It is optional and can be None, which means no prompt depth + will be used. If it is None, the output will be a monocular relative depth. + The values are recommended to be in meters, but this is not necessary. - Returns: - - Examples: + Example: ```python >>> from transformers import AutoImageProcessor, AutoModelForDepthEstimation @@ -483,7 +443,8 @@ class PromptDepthAnythingForDepthEstimation(PromptDepthAnythingPreTrainedModel): >>> depth = predicted_depth * 1000. >>> depth = depth.detach().cpu().numpy() >>> depth = Image.fromarray(depth.astype("uint16")) # mm - ```""" + ``` + """ loss = None if labels is not None: raise NotImplementedError("Training is not implemented yet") diff --git a/src/transformers/models/prompt_depth_anything/modular_prompt_depth_anything.py b/src/transformers/models/prompt_depth_anything/modular_prompt_depth_anything.py index aa834339ea..ae7275a469 100644 --- a/src/transformers/models/prompt_depth_anything/modular_prompt_depth_anything.py +++ b/src/transformers/models/prompt_depth_anything/modular_prompt_depth_anything.py @@ -26,16 +26,9 @@ from transformers.models.depth_anything.modeling_depth_anything import ( ) from transformers.utils.generic import torch_int -from ...file_utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) from ...modeling_outputs import DepthEstimatorOutput from ...modeling_utils import PreTrainedModel - - -_CONFIG_FOR_DOC = "PromptDepthAnythingConfig" +from ...utils import auto_docstring class PromptDepthAnythingConfig(DepthAnythingConfig): @@ -164,45 +157,8 @@ class PromptDepthAnythingDepthEstimationHead(DepthAnythingDepthEstimationHead): return predicted_depth -PROMPT_DEPTH_ANYTHING_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`PromptDepthAnythingConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PROMPT_DEPTH_ANYTHING_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] - for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - prompt_depth (`torch.FloatTensor` of shape `(batch_size, 1, height, width)`, *optional*): - Prompt depth is the sparse or low-resolution depth obtained from multi-view geometry or a - low-resolution depth sensor. It generally has shape (height, width), where height - and width can be smaller than those of the images. It is optional and can be None, which means no prompt depth - will be used. If it is None, the output will be a monocular relative depth. - The values are recommended to be in meters, but this is not necessary. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - +@auto_docstring class PromptDepthAnythingPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = PromptDepthAnythingConfig base_model_prefix = "prompt_depth_anything" main_input_name = "pixel_values" @@ -272,15 +228,12 @@ class PromptDepthAnythingNeck(DepthAnythingNeck): return output -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Prompt Depth Anything Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. - """, - PROMPT_DEPTH_ANYTHING_START_DOCSTRING, + """ ) class PromptDepthAnythingForDepthEstimation(DepthAnythingForDepthEstimation): - @add_start_docstrings_to_model_forward(PROMPT_DEPTH_ANYTHING_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) def forward( self, pixel_values: torch.FloatTensor, @@ -291,6 +244,15 @@ class PromptDepthAnythingForDepthEstimation(DepthAnythingForDepthEstimation): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], DepthEstimatorOutput]: r""" + prompt_depth (`torch.FloatTensor` of shape `(batch_size, 1, height, width)`, *optional*): + Prompt depth is the sparse or low-resolution depth obtained from multi-view geometry or a + low-resolution depth sensor. It generally has shape (height, width), where height + and width can be smaller than those of the images. It is optional and can be None, which means no prompt depth + will be used. If it is None, the output will be a monocular relative depth. + The values are recommended to be in meters, but this is not necessary. + + Example: + ```python >>> from transformers import AutoImageProcessor, AutoModelForDepthEstimation >>> import torch @@ -324,7 +286,8 @@ class PromptDepthAnythingForDepthEstimation(DepthAnythingForDepthEstimation): >>> depth = predicted_depth * 1000. >>> depth = depth.detach().cpu().numpy() >>> depth = Image.fromarray(depth.astype("uint16")) # mm - ```""" + ``` + """ loss = None if labels is not None: raise NotImplementedError("Training is not implemented yet") diff --git a/src/transformers/models/prophetnet/modeling_prophetnet.py b/src/transformers/models/prophetnet/modeling_prophetnet.py index ddb72211b0..0f7cfcd224 100644 --- a/src/transformers/models/prophetnet/modeling_prophetnet.py +++ b/src/transformers/models/prophetnet/modeling_prophetnet.py @@ -29,147 +29,12 @@ from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_prophetnet import ProphetNetConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ProphenetConfig" -_CHECKPOINT_FOR_DOC = "microsoft/prophetnet-large-uncased" - - -PROPHETNET_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - Original ProphetNet code can be found [here](https://github.com/microsoft/ProphetNet). Checkpoints were converted - from original Fairseq checkpoints. For more information on the checkpoint conversion, please take a look at the - file `convert_prophetnet_original_pytorch_checkpoint_to_pytorch.py`. - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matters related to general usage and - behavior. - - Parameters: - config ([`ProphetNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PROPHETNET_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -PROPHETNET_STANDALONE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - def softmax(hidden_state, dim, onnx_trace=False): if onnx_trace: @@ -538,6 +403,7 @@ class ProphetNetDecoderLMOutput(ModelOutput): cross_attentions: Optional[Tuple[torch.FloatTensor]] = None +@auto_docstring class ProphetNetPreTrainedModel(PreTrainedModel): config_class = ProphetNetConfig base_model_prefix = "prophetnet" @@ -1217,18 +1083,18 @@ class ProphetNetDecoderLayer(nn.Module): return outputs -@add_start_docstrings( - "The standalone encoder part of the ProphetNetModel.", - PROPHETNET_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The standalone encoder part of the ProphetNetModel. + """ ) class ProphetNetEncoder(ProphetNetPreTrainedModel): - r""" - word_embeddings (`torch.nn.Embeddings` of shape `(config.vocab_size, config.hidden_size)`, *optional*): - The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with pre-defined word - embeddings instead of randomly initialized word embeddings. - """ - def __init__(self, config: ProphetNetConfig, word_embeddings: nn.Embedding = None): + r""" + word_embeddings (`torch.nn.Embeddings` of shape `(config.vocab_size, config.hidden_size)`, *optional*): + The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with pre-defined word + embeddings instead of randomly initialized word embeddings. + """ super().__init__(config) self.word_embeddings = ( @@ -1251,8 +1117,7 @@ class ProphetNetEncoder(ProphetNetPreTrainedModel): def set_input_embeddings(self, value): self.word_embeddings = value - @add_start_docstrings_to_model_forward(PROPHETNET_STANDALONE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1264,8 +1129,6 @@ class ProphetNetEncoder(ProphetNetPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: r""" - Returns: - Example: ```python @@ -1351,18 +1214,18 @@ class ProphetNetEncoder(ProphetNetPreTrainedModel): ) -@add_start_docstrings( - "The standalone decoder part of the ProphetNetModel.", - PROPHETNET_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The standalone decoder part of the ProphetNetModel. + """ ) class ProphetNetDecoder(ProphetNetPreTrainedModel): - r""" - word_embeddings (`torch.nn.Embeddings` of shape `(config.vocab_size, config.hidden_size)`, *optional*): - The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with pre-defined word - embeddings instead of randomly initialized word embeddings. - """ - def __init__(self, config: ProphetNetConfig, word_embeddings: Optional[nn.Embedding] = None): + r""" + word_embeddings (`torch.nn.Embeddings` of shape `(config.vocab_size, config.hidden_size)`, *optional*): + The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with pre-defined word + embeddings instead of randomly initialized word embeddings. + """ super().__init__(config) self.ngram = config.ngram @@ -1392,8 +1255,7 @@ class ProphetNetDecoder(ProphetNetPreTrainedModel): def set_input_embeddings(self, value): self.word_embeddings = value - @add_start_docstrings_to_model_forward(PROPHETNET_STANDALONE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ProphetNetDecoderModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1410,33 +1272,12 @@ class ProphetNetDecoder(ProphetNetPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ProphetNetDecoderModelOutput]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - Returns: - Example: ```python @@ -1719,10 +1560,7 @@ class ProphetNetDecoder(ProphetNetPreTrainedModel): return extended_predict_attention_mask.to(hidden_states.dtype) -@add_start_docstrings( - "The bare ProphetNet Model outputting raw hidden-states without any specific head on top.", - PROPHETNET_START_DOCSTRING, -) +@auto_docstring class ProphetNetModel(ProphetNetPreTrainedModel): _tied_weights_keys = ["encoder.word_embeddings.weight", "decoder.word_embeddings.weight"] @@ -1762,8 +1600,7 @@ class ProphetNetModel(ProphetNetPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(PROPHETNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ProphetNetSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1783,7 +1620,25 @@ class ProphetNetModel(ProphetNetPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ProphetNetSeq2SeqModelOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1853,9 +1708,10 @@ class ProphetNetModel(ProphetNetPreTrainedModel): ) -@add_start_docstrings( - "The ProphetNet Model with a language modeling head. Can be used for sequence generation tasks.", - PROPHETNET_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The ProphetNet Model with a language modeling head. Can be used for sequence generation tasks. + """ ) class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel, GenerationMixin): _tied_weights_keys = ["encoder.word_embeddings.weight", "decoder.word_embeddings.weight", "lm_head.weight"] @@ -1884,8 +1740,7 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel, GenerationMi def get_input_embeddings(self): return self.prophetnet.word_embeddings - @add_start_docstrings_to_model_forward(PROPHETNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ProphetNetSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1906,13 +1761,30 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel, GenerationMi return_dict: Optional[bool] = None, ) -> Union[Tuple, ProphetNetSeq2SeqLMOutput]: r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Example: ```python @@ -2040,10 +1912,10 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel, GenerationMi return self.prophetnet.decoder -@add_start_docstrings( - "The standalone decoder part of the ProphetNetModel with a lm head on top. The model can be used for causal" - " language modeling.", - PROPHETNET_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The standalone decoder part of the ProphetNetModel with a lm head on top. The model can be used for causal + """ ) class ProphetNetForCausalLM(ProphetNetPreTrainedModel, GenerationMixin): _tied_weights_keys = [ @@ -2090,8 +1962,7 @@ class ProphetNetForCausalLM(ProphetNetPreTrainedModel, GenerationMixin): def get_decoder(self): return self.prophetnet.decoder - @add_start_docstrings_to_model_forward(PROPHETNET_STANDALONE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ProphetNetDecoderLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2109,38 +1980,16 @@ class ProphetNetForCausalLM(ProphetNetPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, ProphetNetDecoderLMOutput]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` - Returns: - Example: ```python diff --git a/src/transformers/models/pvt/image_processing_pvt_fast.py b/src/transformers/models/pvt/image_processing_pvt_fast.py index a371c8b32b..6ad7a155d4 100644 --- a/src/transformers/models/pvt/image_processing_pvt_fast.py +++ b/src/transformers/models/pvt/image_processing_pvt_fast.py @@ -14,18 +14,12 @@ # limitations under the License. """Fast Image processor class for Pvt.""" -from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BaseImageProcessorFast, -) +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast Pvt image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class PvtImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_DEFAULT_MEAN diff --git a/src/transformers/models/pvt/modeling_pvt.py b/src/transformers/models/pvt/modeling_pvt.py index f9eb1a3c58..c4b63ee743 100755 --- a/src/transformers/models/pvt/modeling_pvt.py +++ b/src/transformers/models/pvt/modeling_pvt.py @@ -30,25 +30,12 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, ImageClassifierOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import auto_docstring, logging from .configuration_pvt import PvtConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "PvtConfig" - -_CHECKPOINT_FOR_DOC = "Zetatech/pvt-tiny-224" -_EXPECTED_OUTPUT_SHAPE = [1, 50, 512] - -_IMAGE_CLASS_CHECKPOINT = "Zetatech/pvt-tiny-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: @@ -452,12 +439,8 @@ class PvtEncoder(nn.Module): ) +@auto_docstring class PvtPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = PvtConfig base_model_prefix = "pvt" main_input_name = "pixel_values" @@ -488,37 +471,7 @@ class PvtPreTrainedModel(PreTrainedModel): ) -PVT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~PvtConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PVT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`PvtImageProcessor.__call__`] - for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Pvt encoder outputting raw hidden-states without any specific head on top.", - PVT_START_DOCSTRING, -) +@auto_docstring class PvtModel(PvtPreTrainedModel): def __init__(self, config: PvtConfig): super().__init__(config) @@ -538,14 +491,7 @@ class PvtModel(PvtPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(PVT_INPUTS_DOCSTRING.format("(batch_size, channels, height, width)")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -577,12 +523,11 @@ class PvtModel(PvtPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - PVT_START_DOCSTRING, + """ ) class PvtForImageClassification(PvtPreTrainedModel): def __init__(self, config: PvtConfig) -> None: @@ -599,13 +544,7 @@ class PvtForImageClassification(PvtPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PVT_INPUTS_DOCSTRING.format("(batch_size, channels, height, width)")) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor], diff --git a/src/transformers/models/pvt_v2/modeling_pvt_v2.py b/src/transformers/models/pvt_v2/modeling_pvt_v2.py index 517d2edbe3..7c0fe075b6 100644 --- a/src/transformers/models/pvt_v2/modeling_pvt_v2.py +++ b/src/transformers/models/pvt_v2/modeling_pvt_v2.py @@ -28,27 +28,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput, BaseModelOutput, ImageClassifierOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_pvt_v2 import PvtV2Config logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "PvtV2Config" - -_CHECKPOINT_FOR_DOC = "OpenGVLab/pvt_v2_b0" -_EXPECTED_OUTPUT_SHAPE = [1, 256, 7, 7] - -_IMAGE_CLASS_CHECKPOINT = "OpenGVLab/pvt_v2_b0" -_IMAGE_CLASS_EXPECTED_OUTPUT = "LABEL_281" # ImageNet ID for "tabby, tabby cat" - # Copied from transformers.models.beit.modeling_beit.drop_path def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = False) -> torch.Tensor: @@ -402,12 +388,8 @@ class PvtV2Encoder(nn.Module): ) +@auto_docstring class PvtV2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = PvtV2Config base_model_prefix = "pvt_v2" main_input_name = "pixel_values" @@ -432,37 +414,7 @@ class PvtV2PreTrainedModel(PreTrainedModel): module.bias.data.zero_() -PVT_V2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~PvtV2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -PVT_V2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`PvtImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Pvt-v2 encoder outputting raw hidden-states without any specific head on top.", - PVT_V2_START_DOCSTRING, -) +@auto_docstring class PvtV2Model(PvtV2PreTrainedModel): def __init__(self, config: PvtV2Config): super().__init__(config) @@ -482,14 +434,7 @@ class PvtV2Model(PvtV2PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(PVT_V2_INPUTS_DOCSTRING.format("(batch_size, channels, height, width)")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -521,12 +466,11 @@ class PvtV2Model(PvtV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Pvt-v2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. - """, - PVT_V2_START_DOCSTRING, + """ ) class PvtV2ForImageClassification(PvtV2PreTrainedModel): def __init__(self, config: PvtV2Config) -> None: @@ -543,13 +487,7 @@ class PvtV2ForImageClassification(PvtV2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(PVT_V2_INPUTS_DOCSTRING.format("(batch_size, channels, height, width)")) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor], @@ -621,11 +559,10 @@ class PvtV2ForImageClassification(PvtV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" PVTv2 backbone, to be used with frameworks like DETR and MaskFormer. - """, - PVT_V2_START_DOCSTRING, + """ ) class PvtV2Backbone(PvtV2Model, BackboneMixin): def __init__(self, config: PvtV2Config): @@ -633,8 +570,7 @@ class PvtV2Backbone(PvtV2Model, BackboneMixin): super()._init_backbone(config) self.num_features = config.hidden_sizes - @add_start_docstrings_to_model_forward(PVT_V2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -642,9 +578,7 @@ class PvtV2Backbone(PvtV2Model, BackboneMixin): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/qwen2/modeling_qwen2.py b/src/transformers/models/qwen2/modeling_qwen2.py index ea70a06ee3..2ddff4d203 100644 --- a/src/transformers/models/qwen2/modeling_qwen2.py +++ b/src/transformers/models/qwen2/modeling_qwen2.py @@ -26,16 +26,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_qwen2 import Qwen2Config @@ -47,9 +38,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "meta-qwen2/Qwen2-2-7b-hf" -_CONFIG_FOR_DOC = "Qwen2Config" - class Qwen2MLP(nn.Module): def __init__(self, config): @@ -292,27 +280,7 @@ class Qwen2DecoderLayer(GradientCheckpointingLayer): return outputs -QWEN2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2 Model outputting raw hidden-states without any specific head on top.", - QWEN2_START_DOCSTRING, -) +@auto_docstring class Qwen2PreTrainedModel(PreTrainedModel): config_class = Qwen2Config base_model_prefix = "model" @@ -375,88 +343,8 @@ class Qwen2RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -QWEN2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Qwen2 Model outputting raw hidden-states without any specific head on top.", - QWEN2_START_DOCSTRING, -) +@auto_docstring class Qwen2Model(Qwen2PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`] - - Args: - config: Qwen2Config - """ - def __init__(self, config: Qwen2Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -480,7 +368,7 @@ class Qwen2Model(Qwen2PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -733,6 +621,7 @@ class Qwen2Model(Qwen2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Qwen2ForCausalLM(Qwen2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -766,8 +655,7 @@ class Qwen2ForCausalLM(Qwen2PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -784,19 +672,10 @@ class Qwen2ForCausalLM(Qwen2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -851,8 +730,8 @@ class Qwen2ForCausalLM(Qwen2PreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Qwen2 Model transformer with a sequence classification head on top (linear layer). [`Qwen2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -863,8 +742,7 @@ class Qwen2ForCausalLM(Qwen2PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - QWEN2_START_DOCSTRING, + """ ) class Qwen2ForSequenceClassification(Qwen2PreTrainedModel): def __init__(self, config): @@ -883,7 +761,7 @@ class Qwen2ForSequenceClassification(Qwen2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -952,13 +830,7 @@ class Qwen2ForSequenceClassification(Qwen2PreTrainedModel): ) -@add_start_docstrings( - """ - The Qwen2 Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - QWEN2_START_DOCSTRING, -) +@auto_docstring class Qwen2ForTokenClassification(Qwen2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -983,12 +855,7 @@ class Qwen2ForTokenClassification(Qwen2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1034,13 +901,7 @@ class Qwen2ForTokenClassification(Qwen2PreTrainedModel): ) -@add_start_docstrings( - """ -The Qwen2 Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - QWEN2_START_DOCSTRING, -) +@auto_docstring class Qwen2ForQuestionAnswering(Qwen2PreTrainedModel): base_model_prefix = "transformer" @@ -1059,7 +920,7 @@ class Qwen2ForQuestionAnswering(Qwen2PreTrainedModel): self.transformer.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1073,17 +934,6 @@ class Qwen2ForQuestionAnswering(Qwen2PreTrainedModel): output_hidden_states: Optional[bool] = None, **kwargs, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.transformer( input_ids, attention_mask=attention_mask, diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index b1b875abcd..4849fe4e20 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -39,14 +39,12 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, ModelO from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, check_torch_load_is_safe, is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10, is_torch_flex_attn_available, logging, - replace_return_docstrings, ) from ...utils.hub import cached_file from .configuration_qwen2_5_omni import ( @@ -103,27 +101,7 @@ class Qwen2RMSNorm(nn.Module): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -Qwen2_5Omni_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2_5OmniConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2_5Omni Model outputting raw hidden-states without any specific head on top.", - Qwen2_5Omni_START_DOCSTRING, -) +@auto_docstring class Qwen2_5OmniPreTrainedModel(PreTrainedModel): config_class = Qwen2_5OmniConfig base_model_prefix = "model" @@ -828,30 +806,13 @@ class SinusoidsPositionEmbedding(nn.Module): return self.positional_embedding[:seqlen, :] -QWEN_OMNI_AUDIO_INPUTS = r""" - Args: - input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - - feature_lens: [B], torch.LongTensor , mel length - - aftercnn_lens : [B], torch.LongTensor , mel length after cnn - """ - - -class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): - """ +@auto_docstring( + custom_intro=""" Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`Qwen2_5OmniAudioEncoderLayer`]. - - Args: - config: Qwen2_5OmniAudioEncoderConfig """ - +) +class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniAudioEncoderConfig main_input_name = "input_features" _no_split_modules = ["Qwen2_5OmniAudioEncoderLayer"] @@ -889,13 +850,25 @@ class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): def set_input_embeddings(self, value: nn.Module): self.conv1 = value - @add_start_docstrings_to_model_forward(QWEN_OMNI_AUDIO_INPUTS) + @auto_docstring def forward( self, input_features, feature_lens=None, aftercnn_lens=None, ): + r""" + input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + feature_lens (`torch.LongTensor` of shape `(batch_size,)`): + mel length + aftercnn_lens (`torch.LongTensor` of shape `(batch_size,)`): + mel length after cnn + """ chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long() chunk_lengths = torch.tensor( @@ -1877,27 +1850,7 @@ class Qwen2_5OmniDecoderLayer(nn.Module): return outputs -QWEN2_5OMNI_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`{config_class}`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2.5OmniThinker Model outputting raw hidden-states without any specific head on top.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniTextConfig"), -) +@auto_docstring class Qwen2_5OmniThinkerTextModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniTextConfig _no_split_modules = ["Qwen2_5OmniDecoderLayer"] @@ -1925,6 +1878,7 @@ class Qwen2_5OmniThinkerTextModel(Qwen2_5OmniPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2197,98 +2151,10 @@ class Qwen2_5OmniThinkerTextModel(Qwen2_5OmniPreTrainedModel): return causal_mask -QWEN2_5OMNITHINKER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, feature_sequence_length)`): - Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by - loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via - the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size), *optional*): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`NewTaskModelProcessor`] uses - [`SiglipImageProcessor`] for processing images). - pixel_values_videos(`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size), *optional*): - The tensors corresponding to the input videos. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`NewTaskModelProcessor`] uses - [`SiglipImageProcessor`] for processing videos). - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`, *optional*): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): - The length of feature shape of each audio in LLM. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`list(torch.FloatTensor)`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """The Qwen2.5OmniThinker model which consists of a audio backbone and a language model.""", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniThinkerConfig"), +@auto_docstring( + custom_intro=""" + The Qwen2.5OmniThinker model which consists of a audio backbone and a language model. + """ ) class Qwen2_5OmniThinkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForConditionalGeneration, GenerationMixin): config_class = Qwen2_5OmniThinkerConfig @@ -2321,10 +2187,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCo def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(QWEN2_5OMNITHINKER_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=Qwen2_5OmniThinkerCausalLMOutputWithPast, config_class="Qwen2_5OmniThinkerConfig" - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2350,13 +2213,37 @@ class Qwen2_5OmniThinkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCo video_second_per_grid: Optional[torch.LongTensor] = None, ) -> Union[Tuple, Qwen2_5OmniThinkerCausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, feature_sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size), *optional*): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`NewTaskModelProcessor`] uses + [`SiglipImageProcessor`] for processing videos). + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`, *optional*): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - Returns: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_audio_in_video (`bool`, *optional*): + Whether or not use audio track in video, should same as the parameter in `process_audio_info`. + video_second_per_grid (`torch.LongTensor` of shape `(num_videos)`, *optional*): + Number of seconds per grid for each video, used for temporal feature mapping. Example: @@ -2611,10 +2498,7 @@ class Qwen2_5OmniTalkerCausalLMOutputWithPast(ModelOutput): thinker_reply_part: torch.FloatTensor = None -@add_start_docstrings( - "The bare Qwen2.5OmniTalker Model outputting raw hidden-states without any specific head on top.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniTalkerConfig"), -) +@auto_docstring class Qwen2_5OmniTalkerModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniTalkerConfig _no_split_modules = ["Qwen2_5OmniTalkerDecoderLayer"] @@ -2641,6 +2525,7 @@ class Qwen2_5OmniTalkerModel(Qwen2_5OmniPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2946,10 +2831,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCon def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(QWEN2_5OMNITHINKER_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=Qwen2_5OmniTalkerCausalLMOutputWithPast, config_class="Qwen2_5OmniTalkerConfig" - ) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -2972,13 +2854,22 @@ class Qwen2_5OmniTalkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCon return_dict: Optional[bool] = None, ) -> Union[Tuple, Qwen2_5OmniTalkerCausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. + thinker_reply_part (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Hidden states from the thinker model's output that represent the text reply part to be processed. + input_text_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Input token IDs for text-only content, used for position calculation in multimodal contexts. + use_audio_in_video (`bool`, *optional*): + Whether or not use audio track in video, should same as the parameter in `process_audio_info`. + video_second_per_grid (`torch.LongTensor` of shape `(num_videos)`, *optional*): + Number of seconds per grid for each video, used for temporal feature mapping. Example: @@ -4006,9 +3897,10 @@ class AMPBlock(torch.nn.Module): return hidden_states -@add_start_docstrings( - "The full Qwen2.5Omni Token2WavBigVGAN model. Which take mel spectrogram as input and predict waveform.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniBigVGANConfig"), +@auto_docstring( + custom_intro=""" + The full Qwen2.5Omni Token2WavBigVGAN model. Which take mel spectrogram as input and predict waveform. + """ ) class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniBigVGANConfig @@ -4140,9 +4032,10 @@ class RungeKutta4ODESolver: return solution -@add_start_docstrings( - "The full Qwen2.5Omni Token2WavDiT model. Which take speech tokens as input and predict mel spectrogram.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniDiTConfig"), +@auto_docstring( + custom_intro=""" + The full Qwen2.5Omni Token2WavDiT model. Which take speech tokens as input and predict mel spectrogram. + """ ) class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniDiTConfig @@ -4294,9 +4187,10 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): return generated_mel_spectrogram -@add_start_docstrings( - "The full Qwen2.5Omni Token2Wav model. Consists a DiT model take speech tokens as input and predict mel spectrogram and a BigVGAN vocoder take mel spectrogram as input and predict waveform.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniToken2WavConfig"), +@auto_docstring( + custom_intro=""" + The full Qwen2.5Omni Token2Wav model. Consists a DiT model take speech tokens as input and predict mel spectrogram and a BigVGAN vocoder take mel spectrogram as input and predict waveform. + """ ) class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniToken2WavConfig @@ -4355,8 +4249,8 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel): ############################ -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The full Qwen2.5Omni model, a multimodal model composed of 3 sub-models: - [`Qwen2_5OmniThinkerForConditionalGeneration`]: a causal auto-regressive transformer takes text, audio, image, video as input and predict text tokens. @@ -4364,8 +4258,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel): a causal auto-regressive transformer takes thinker hidden states and response as input and predict speech tokens. - [`Qwen2_5OmniToken2WavModel`]: a DiT model take speech tokens as input and predict mel spectrogram and a BigVGAN vocoder take mel spectrogram as input and predict waveform. - """, - QWEN2_5OMNI_START_DOCSTRING.format(config_class=Qwen2_5OmniConfig), + """ ) class Qwen2_5OmniForConditionalGeneration(Qwen2_5OmniPreTrainedModel, GenerationMixin): config_class = Qwen2_5OmniConfig diff --git a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py index 5a51e78757..b1a90d9969 100644 --- a/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py @@ -47,13 +47,11 @@ from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, check_torch_load_is_safe, is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10, logging, - replace_return_docstrings, ) from ...utils.hub import cached_file @@ -1724,30 +1722,13 @@ class SinusoidsPositionEmbedding(nn.Module): return self.positional_embedding[:seqlen, :] -QWEN_OMNI_AUDIO_INPUTS = r""" - Args: - input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): - Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be - obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a - `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into - `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding - and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - - feature_lens: [B], torch.LongTensor , mel length - - aftercnn_lens : [B], torch.LongTensor , mel length after cnn - """ - - -class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): - """ +@auto_docstring( + custom_intro=""" Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`Qwen2_5OmniAudioEncoderLayer`]. - - Args: - config: Qwen2_5OmniAudioEncoderConfig """ - +) +class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniAudioEncoderConfig main_input_name = "input_features" _no_split_modules = ["Qwen2_5OmniAudioEncoderLayer"] @@ -1785,13 +1766,25 @@ class Qwen2_5OmniAudioEncoder(Qwen2_5OmniPreTrainedModel): def set_input_embeddings(self, value: nn.Module): self.conv1 = value - @add_start_docstrings_to_model_forward(QWEN_OMNI_AUDIO_INPUTS) + @auto_docstring def forward( self, input_features, feature_lens=None, aftercnn_lens=None, ): + r""" + input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be + obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a + `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into + `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding + and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + feature_lens (`torch.LongTensor` of shape `(batch_size,)`): + mel length + aftercnn_lens (`torch.LongTensor` of shape `(batch_size,)`): + mel length after cnn + """ chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long() chunk_lengths = torch.tensor( @@ -2144,27 +2137,6 @@ class Qwen2MLP(Qwen2_5_VLMLP): pass -QWEN2_5OMNI_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`{config_class}`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2.5OmniThinker Model outputting raw hidden-states without any specific head on top.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniTextConfig"), -) class Qwen2_5OmniThinkerTextModel(Qwen2_5_VLTextModel): config_class = Qwen2_5OmniTextConfig _no_split_modules = ["Qwen2_5OmniDecoderLayer"] @@ -2173,98 +2145,10 @@ class Qwen2_5OmniThinkerTextModel(Qwen2_5_VLTextModel): super().__init__(config) -QWEN2_5OMNITHINKER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, feature_sequence_length)`): - Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by - loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via - the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size), *optional*): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`NewTaskModelProcessor`] uses - [`SiglipImageProcessor`] for processing images). - pixel_values_videos(`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size), *optional*): - The tensors corresponding to the input videos. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`NewTaskModelProcessor`] uses - [`SiglipImageProcessor`] for processing videos). - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`, *optional*): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): - The length of feature shape of each audio in LLM. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`list(torch.FloatTensor)`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """The Qwen2.5OmniThinker model which consists of a audio backbone and a language model.""", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniThinkerConfig"), +@auto_docstring( + custom_intro=""" + The Qwen2.5OmniThinker model which consists of a audio backbone and a language model. + """ ) class Qwen2_5OmniThinkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForConditionalGeneration, GenerationMixin): config_class = Qwen2_5OmniThinkerConfig @@ -2297,10 +2181,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCo def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(QWEN2_5OMNITHINKER_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=Qwen2_5OmniThinkerCausalLMOutputWithPast, config_class="Qwen2_5OmniThinkerConfig" - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2326,13 +2207,37 @@ class Qwen2_5OmniThinkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCo video_second_per_grid: Optional[torch.LongTensor] = None, ) -> Union[Tuple, Qwen2_5OmniThinkerCausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, feature_sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size), *optional*): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`SiglipImageProcessor.__call__`] for details ([]`NewTaskModelProcessor`] uses + [`SiglipImageProcessor`] for processing videos). + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`, *optional*): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - Returns: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_audio_in_video (`bool`, *optional*): + Whether or not use audio track in video, should same as the parameter in `process_audio_info`. + video_second_per_grid (`torch.LongTensor` of shape `(num_videos)`, *optional*): + Number of seconds per grid for each video, used for temporal feature mapping. Example: @@ -2587,10 +2492,6 @@ class Qwen2_5OmniTalkerCausalLMOutputWithPast(ModelOutput): thinker_reply_part: torch.FloatTensor = None -@add_start_docstrings( - "The bare Qwen2.5OmniTalker Model outputting raw hidden-states without any specific head on top.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniTalkerConfig"), -) class Qwen2_5OmniTalkerModel(Qwen2_5_VLTextModel): config_class = Qwen2_5OmniTalkerConfig _no_split_modules = ["Qwen2_5OmniTalkerDecoderLayer"] @@ -2633,10 +2534,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCon def set_input_embeddings(self, value): self.model.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(QWEN2_5OMNITHINKER_INPUTS_DOCSTRING) - @replace_return_docstrings( - output_type=Qwen2_5OmniTalkerCausalLMOutputWithPast, config_class="Qwen2_5OmniTalkerConfig" - ) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -2659,13 +2557,22 @@ class Qwen2_5OmniTalkerForConditionalGeneration(Qwen2_5OmniPreTrainedModelForCon return_dict: Optional[bool] = None, ) -> Union[Tuple, Qwen2_5OmniTalkerCausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*): + The length of feature shape of each audio in LLM. + thinker_reply_part (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Hidden states from the thinker model's output that represent the text reply part to be processed. + input_text_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Input token IDs for text-only content, used for position calculation in multimodal contexts. + use_audio_in_video (`bool`, *optional*): + Whether or not use audio track in video, should same as the parameter in `process_audio_info`. + video_second_per_grid (`torch.LongTensor` of shape `(num_videos)`, *optional*): + Number of seconds per grid for each video, used for temporal feature mapping. Example: @@ -3693,9 +3600,10 @@ class AMPBlock(torch.nn.Module): return hidden_states -@add_start_docstrings( - "The full Qwen2.5Omni Token2WavBigVGAN model. Which take mel spectrogram as input and predict waveform.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniBigVGANConfig"), +@auto_docstring( + custom_intro=""" + The full Qwen2.5Omni Token2WavBigVGAN model. Which take mel spectrogram as input and predict waveform. + """ ) class Qwen2_5OmniToken2WavBigVGANModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniBigVGANConfig @@ -3827,9 +3735,10 @@ class RungeKutta4ODESolver: return solution -@add_start_docstrings( - "The full Qwen2.5Omni Token2WavDiT model. Which take speech tokens as input and predict mel spectrogram.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniDiTConfig"), +@auto_docstring( + custom_intro=""" + The full Qwen2.5Omni Token2WavDiT model. Which take speech tokens as input and predict mel spectrogram. + """ ) class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniDiTConfig @@ -3981,9 +3890,10 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): return generated_mel_spectrogram -@add_start_docstrings( - "The full Qwen2.5Omni Token2Wav model. Consists a DiT model take speech tokens as input and predict mel spectrogram and a BigVGAN vocoder take mel spectrogram as input and predict waveform.", - QWEN2_5OMNI_START_DOCSTRING.format(config_class="Qwen2_5OmniToken2WavConfig"), +@auto_docstring( + custom_intro=""" + The full Qwen2.5Omni Token2Wav model. Consists a DiT model take speech tokens as input and predict mel spectrogram and a BigVGAN vocoder take mel spectrogram as input and predict waveform. + """ ) class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel): config_class = Qwen2_5OmniToken2WavConfig @@ -4042,8 +3952,8 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel): ############################ -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The full Qwen2.5Omni model, a multimodal model composed of 3 sub-models: - [`Qwen2_5OmniThinkerForConditionalGeneration`]: a causal auto-regressive transformer takes text, audio, image, video as input and predict text tokens. @@ -4051,8 +3961,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel): a causal auto-regressive transformer takes thinker hidden states and response as input and predict speech tokens. - [`Qwen2_5OmniToken2WavModel`]: a DiT model take speech tokens as input and predict mel spectrogram and a BigVGAN vocoder take mel spectrogram as input and predict waveform. - """, - QWEN2_5OMNI_START_DOCSTRING.format(config_class=Qwen2_5OmniConfig), + """ ) class Qwen2_5OmniForConditionalGeneration(Qwen2_5OmniPreTrainedModel, GenerationMixin): config_class = Qwen2_5OmniConfig diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index f87260c69b..fa2aa76159 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -40,14 +40,7 @@ from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_qwen2_5_vl import Qwen2_5_VLConfig, Qwen2_5_VLTextConfig, Qwen2_5_VLVisionConfig @@ -66,8 +59,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Qwen2_5_VLConfig" - class Qwen2_5_VLMLP(nn.Module): def __init__(self, config, bias: bool = False): @@ -357,27 +348,7 @@ class Qwen2_5_VLVisionBlock(nn.Module): return hidden_states -Qwen2_5_VL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2_5_VLConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2_5_VL Model outputting raw hidden-states without any specific head on top.", - Qwen2_5_VL_START_DOCSTRING, -) +@auto_docstring class Qwen2_5_VLPreTrainedModel(PreTrainedModel): config_class = Qwen2_5_VLConfig base_model_prefix = "model" @@ -1108,10 +1079,7 @@ class Qwen2_5_VLDecoderLayer(nn.Module): return outputs -@add_start_docstrings( - "The bare Qwen2_5_VL Model outputting raw hidden-states without any specific head on top.", - Qwen2_5_VL_START_DOCSTRING, -) +@auto_docstring class Qwen2_5_VLTextModel(Qwen2_5_VLPreTrainedModel): config_class = Qwen2_5_VLTextConfig @@ -1138,6 +1106,7 @@ class Qwen2_5_VLTextModel(Qwen2_5_VLPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1410,82 +1379,7 @@ class Qwen2_5_VLTextModel(Qwen2_5_VLPreTrainedModel): return causal_mask -QWEN2_5_VL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - pixel_values (`torch.FloatTensor` of shape `(seq_length, num_channels * image_size * image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`Qwen2_5_VLImageProcessor.__call__`] for details. [`Qwen2_5_VLProcessor`] uses - [`Qwen2_5_VLImageProcessor`] for processing images. - pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): - The tensors corresponding to the input videos. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`Qwen2_5_VLImageProcessor.__call__`] for details. [`Qwen2_5_VLProcessor`] uses - [`Qwen2_5_VLImageProcessor`] for processing videos. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. -""" - - +@auto_docstring class Qwen2_5_VLModel(Qwen2_5_VLPreTrainedModel): _checkpoint_conversion_mapping = {"^model": "language_model"} config_class = Qwen2_5_VLConfig @@ -1688,7 +1582,7 @@ class Qwen2_5_VLModel(Qwen2_5_VLPreTrainedModel): return position_ids, mrope_position_deltas - @add_start_docstrings_to_model_forward(QWEN2_5_VL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1708,6 +1602,21 @@ class Qwen2_5_VLModel(Qwen2_5_VLPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, second_per_grid_ts: Optional[torch.Tensor] = None, ) -> Union[Tuple, Qwen2_5_VLModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2_5_VLImageProcessor.__call__`] for details. [`Qwen2_5_VLProcessor`] uses + [`Qwen2_5_VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1890,8 +1799,7 @@ class Qwen2_5_VLForConditionalGeneration(Qwen2_5_VLPreTrainedModel, GenerationMi return self.model.visual @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_5_VL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Qwen2_5_VLCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1913,12 +1821,22 @@ class Qwen2_5_VLForConditionalGeneration(Qwen2_5_VLPreTrainedModel, GenerationMi second_per_grid_ts: Optional[torch.Tensor] = None, ) -> Union[Tuple, Qwen2_5_VLCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2_5_VLImageProcessor.__call__`] for details. [`Qwen2_5_VLProcessor`] uses + [`Qwen2_5_VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. Example: diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index 2d220aa2a9..6a8e3080a6 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -50,12 +50,7 @@ from ...image_utils import ImageInput, VideoInput from ...modeling_flash_attention_utils import is_flash_attn_available from ...processing_utils import ProcessingKwargs, Unpack, VideosKwargs from ...tokenization_utils_base import PreTokenizedInput, TextInput -from ...utils import ( - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import logging if is_flash_attn_available(): @@ -64,8 +59,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Qwen2_5_VLConfig" - def apply_rotary_pos_emb_flashatt( q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor @@ -627,6 +620,21 @@ class Qwen2_5_VLModel(Qwen2VLModel): cache_position: Optional[torch.LongTensor] = None, second_per_grid_ts: Optional[torch.Tensor] = None, ) -> Union[Tuple, Qwen2_5_VLModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2_5_VLImageProcessor.__call__`] for details. [`Qwen2_5_VLProcessor`] uses + [`Qwen2_5_VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -733,13 +741,7 @@ class Qwen2_5_VLCausalLMOutputWithPast(Qwen2VLCausalLMOutputWithPast): pass -QWEN2_5_VL_INPUTS_DOCSTRING = None - - class Qwen2_5_VLForConditionalGeneration(Qwen2VLForConditionalGeneration): - @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_5_VL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Qwen2_5_VLCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: torch.LongTensor = None, @@ -761,12 +763,22 @@ class Qwen2_5_VLForConditionalGeneration(Qwen2VLForConditionalGeneration): second_per_grid_ts: Optional[torch.Tensor] = None, ) -> Union[Tuple, Qwen2_5_VLCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2_5_VLImageProcessor.__call__`] for details. [`Qwen2_5_VLProcessor`] uses + [`Qwen2_5_VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. Example: diff --git a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py index 03700451ce..b71fe61813 100644 --- a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py +++ b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py @@ -28,12 +28,7 @@ from ...generation import GenerationMixin from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.deprecation import deprecate_kwarg from ..auto import AutoModel, AutoModelForCausalLM from .configuration_qwen2_audio import Qwen2AudioConfig, Qwen2AudioEncoderConfig @@ -45,8 +40,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Qwen2AudioConfig" - @dataclass class Qwen2AudioCausalLMOutputWithPast(ModelOutput): @@ -438,27 +431,7 @@ class Qwen2AudioEncoderLayer(nn.Module): return outputs -QWEN2AUDIO_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2AudioConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2Audio Model outputting raw hidden-states without any specific head on top.", - QWEN2AUDIO_START_DOCSTRING, -) +@auto_docstring class Qwen2AudioPreTrainedModel(PreTrainedModel): config_class = Qwen2AudioConfig base_model_prefix = "model" @@ -487,26 +460,10 @@ class Qwen2AudioPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -QWEN2AUDIOENCODER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2AudioEncoderConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The audio model from Qwen2Audio without any head or projection on top.""", - QWEN2AUDIOENCODER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The audio model from Qwen2Audio without any head or projection on top. + """ ) # Copied from transformers.models.whisper.modeling_whisper.WhisperEncoder with Whisper->Qwen2Audio class Qwen2AudioEncoder(Qwen2AudioPreTrainedModel): @@ -698,81 +655,10 @@ class Qwen2AudioMultiModalProjector(nn.Module): return hidden_states -QWEN2AUDIO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, feature_sequence_length)`): - Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by - loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via - the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): - Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Pre-computed hidden-states that can be used to speed up auto-regressive (sequential) decoding. There are - two sets of pre-computed hidden-states: key and values states in the self-attention blocks. - The `past_key_values` are returned when `use_cache=True` is passed or when `config.use_cache=True`. - It is a [`~cache_utils.Cache`] instance. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `input_ids` of shape `(batch_size, sequence_length)`.shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """The QWEN2AUDIO model which consists of a audio backbone and a language model.""", - QWEN2AUDIO_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The QWEN2AUDIO model which consists of a audio backbone and a language model. + """ ) class Qwen2AudioForConditionalGeneration(Qwen2AudioPreTrainedModel, GenerationMixin): def __init__(self, config: Qwen2AudioConfig): @@ -1014,8 +900,7 @@ class Qwen2AudioForConditionalGeneration(Qwen2AudioPreTrainedModel, GenerationMi return final_embedding, final_attention_mask, final_labels, position_ids, final_input_ids - @add_start_docstrings_to_model_forward(QWEN2AUDIO_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Qwen2AudioCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1032,12 +917,21 @@ class Qwen2AudioForConditionalGeneration(Qwen2AudioPreTrainedModel, GenerationMi return_dict: Optional[bool] = None, ) -> Union[Tuple, Qwen2AudioCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, feature_sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`): + Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`: - Returns: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py index 00cb7896fd..7bc3606c5b 100644 --- a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py @@ -41,15 +41,7 @@ from ...modeling_outputs import ( ) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_qwen2_moe import Qwen2MoeConfig @@ -63,9 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Qwen/Qwen2-57B-A14B" -_CONFIG_FOR_DOC = "Qwen2MoeConfig" - # Copied from transformers.models.mixtral.modeling_mixtral.load_balancing_loss_func def load_balancing_loss_func( @@ -743,27 +732,7 @@ class Qwen2MoeDecoderLayer(nn.Module): return outputs -QWEN2MOE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2MoeConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2MoE Model outputting raw hidden-states without any specific head on top.", - QWEN2MOE_START_DOCSTRING, -) +@auto_docstring class Qwen2MoePreTrainedModel(PreTrainedModel): config_class = Qwen2MoeConfig base_model_prefix = "model" @@ -788,96 +757,8 @@ class Qwen2MoePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -QWEN2MOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Qwen2MoE Model outputting raw hidden-states without any specific head on top.", - QWEN2MOE_START_DOCSTRING, -) +@auto_docstring class Qwen2MoeModel(Qwen2MoePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2MoeDecoderLayer`] - - Args: - config: Qwen2MoeConfig - """ - def __init__(self, config: Qwen2MoeConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -902,7 +783,7 @@ class Qwen2MoeModel(Qwen2MoePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2MOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1227,8 +1108,7 @@ class Qwen2MoeForCausalLM(Qwen2MoePreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2MOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoeCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1246,19 +1126,10 @@ class Qwen2MoeForCausalLM(Qwen2MoePreTrainedModel, GenerationMixin): **loss_kwargs, ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1330,8 +1201,8 @@ class Qwen2MoeForCausalLM(Qwen2MoePreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Qwen2MoE Model transformer with a sequence classification head on top (linear layer). [`Qwen2MoeForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1342,8 +1213,7 @@ class Qwen2MoeForCausalLM(Qwen2MoePreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - QWEN2MOE_START_DOCSTRING, + """ ) # Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Qwen2Moe, LLAMA->QWEN2MOE, BaseModelOutputWithPast->MoeModelOutputWithPast class Qwen2MoeForSequenceClassification(Qwen2MoePreTrainedModel): @@ -1363,7 +1233,7 @@ class Qwen2MoeForSequenceClassification(Qwen2MoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2MOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1432,13 +1302,7 @@ class Qwen2MoeForSequenceClassification(Qwen2MoePreTrainedModel): ) -@add_start_docstrings( - """ - The Qwen2MoE Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - QWEN2MOE_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.llama.modeling_llama.LlamaForTokenClassification with Llama->Qwen2Moe, LLAMA->QWEN2MOE, BaseModelOutputWithPast->MoeModelOutputWithPast class Qwen2MoeForTokenClassification(Qwen2MoePreTrainedModel): def __init__(self, config): @@ -1464,12 +1328,7 @@ class Qwen2MoeForTokenClassification(Qwen2MoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2MOE_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1515,13 +1374,7 @@ class Qwen2MoeForTokenClassification(Qwen2MoePreTrainedModel): ) -@add_start_docstrings( - """ -The Qwen2MoE Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - QWEN2MOE_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.mistral.modeling_mistral.MistralForQuestionAnswering with Mistral->Qwen2Moe, MISTRAL->QWEN2MOE, BaseModelOutputWithPast->MoeModelOutputWithPast class Qwen2MoeForQuestionAnswering(Qwen2MoePreTrainedModel): base_model_prefix = "model" @@ -1541,7 +1394,7 @@ class Qwen2MoeForQuestionAnswering(Qwen2MoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2MOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/qwen2_vl/image_processing_qwen2_vl_fast.py b/src/transformers/models/qwen2_vl/image_processing_qwen2_vl_fast.py index 350d12ca87..ef7fd4f280 100644 --- a/src/transformers/models/qwen2_vl/image_processing_qwen2_vl_fast.py +++ b/src/transformers/models/qwen2_vl/image_processing_qwen2_vl_fast.py @@ -23,7 +23,6 @@ from typing import Dict, List, Optional, Union from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -45,7 +44,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -70,6 +69,19 @@ logger = logging.get_logger(__name__) class Qwen2VLFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + min_pixels (`int`, *optional*, defaults to `56 * 56`): + The min pixels of the image to resize the image. + max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`): + The max pixels of the image to resize the image. + patch_size (`int`, *optional*, defaults to 14): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 2): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The merge size of the vision encoder to llm encoder. + """ + min_pixels: Optional[int] max_pixels: Optional[int] patch_size: Optional[int] @@ -77,22 +89,7 @@ class Qwen2VLFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): merge_size: Optional[int] -@add_start_docstrings( - "Constructs a fast Qwen2-VL image processor that dynamically resizes images based on the original images.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - min_pixels (`int`, *optional*, defaults to `56 * 56`): - The min pixels of the image to resize the image. - max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`): - The max pixels of the image to resize the image. - patch_size (`int`, *optional*, defaults to 14): - The spatial patch size of the vision encoder. - temporal_patch_size (`int`, *optional*, defaults to 2): - The temporal patch size of the vision encoder. - merge_size (`int`, *optional*, defaults to 2): - The merge size of the vision encoder to llm encoder. - """, -) +@auto_docstring class Qwen2VLImageProcessorFast(BaseImageProcessorFast): do_resize = True resample = PILImageResampling.BICUBIC @@ -251,6 +248,7 @@ class Qwen2VLImageProcessorFast(BaseImageProcessorFast): return flatten_patches, (grid_t, grid_h, grid_w) + @auto_docstring def preprocess( self, images: ImageInput, @@ -275,64 +273,17 @@ class Qwen2VLImageProcessorFast(BaseImageProcessorFast): device: Optional["torch.device"] = None, **kwargs, ): - """ - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - videos (`VideoInput`): - Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If - passing in videos with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - size (`Dict[str, int]`, *optional*, defaults to `self.size`): - Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present. - resample (`int`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only - has an effect if `do_resize` is set to `True`. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Rescale factor to rescale the image by if `do_rescale` is set to `True`. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): - Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): - Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to - `True`. - min_pixels (`int`, *optional*, defaults to `self.min_pixels`): - The min pixels of the image to resize the image. - max_pixels (`int`, *optional*, defaults to `self.max_pixels`): - The max pixels of the image to resize the image. - patch_size (`int`, *optional*, defaults to `self.patch_size`): - The spatial patch size of the vision encoder. - temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`): - The temporal patch size of the vision encoder. - merge_size (`int`, *optional*, defaults to `self.merge_size`): - The merge size of the vision encoder to llm encoder. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - device (`torch.device`, *optional*): - The device to process the images on. If unset, the device is inferred from the input images. + r""" + min_pixels (`int`, *optional*, defaults to `56 * 56`): + The min pixels of the image to resize the image. + max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`): + The max pixels of the image to resize the image. + patch_size (`int`, *optional*, defaults to 14): + The spatial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 2): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The merge size of the vision encoder to llm encoder. """ min_pixels = min_pixels if min_pixels is not None else self.min_pixels max_pixels = max_pixels if max_pixels is not None else self.max_pixels diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 2e46c8927a..ca9783cc4c 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -37,15 +37,7 @@ from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, is_torchdynamo_compiling, logging from .configuration_qwen2_vl import Qwen2VLConfig, Qwen2VLTextConfig, Qwen2VLVisionConfig @@ -60,8 +52,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Qwen2VLConfig" - @dataclass class Qwen2VLModelOutputWithPast(ModelOutput): @@ -924,27 +914,7 @@ class Qwen2VLDecoderLayer(nn.Module): return outputs -QWEN2VL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen2VLConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen2VL Model outputting raw hidden-states without any specific head on top.", - QWEN2VL_START_DOCSTRING, -) +@auto_docstring class Qwen2VLPreTrainedModel(PreTrainedModel): config_class = Qwen2VLConfig base_model_prefix = "model" @@ -973,6 +943,7 @@ class Qwen2VLPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) +@auto_docstring class Qwen2VisionTransformerPretrainedModel(Qwen2VLPreTrainedModel): config_class = Qwen2VLVisionConfig _no_split_modules = ["Qwen2VLVisionBlock"] @@ -1034,7 +1005,12 @@ class Qwen2VisionTransformerPretrainedModel(Qwen2VLPreTrainedModel): rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb + @auto_docstring def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + r""" + grid_thw (`torch.LongTensor` of shape `(num_images, 3)`): + The temporal, height and width dimensions of feature shape for each image. Each row contains [t, h, w] values. + """ hidden_states = self.patch_embed(hidden_states) rotary_pos_emb = self.rot_pos_emb(grid_thw) emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) @@ -1061,10 +1037,7 @@ class Qwen2VisionTransformerPretrainedModel(Qwen2VLPreTrainedModel): return self.merger(hidden_states) -@add_start_docstrings( - "The bare Qwen2VL Model outputting raw hidden-states without any specific head on top.", - QWEN2VL_START_DOCSTRING, -) +@auto_docstring class Qwen2VLTextModel(Qwen2VLPreTrainedModel): config_class = Qwen2VLTextConfig @@ -1091,6 +1064,7 @@ class Qwen2VLTextModel(Qwen2VLPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1365,82 +1339,7 @@ class Qwen2VLTextModel(Qwen2VLPreTrainedModel): return causal_mask -QWEN2_VL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - pixel_values (`torch.FloatTensor` of shape `(seq_length, num_channels * image_size * image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`Qwen2VLImageProcessor.__call__`] for details. [`Qwen2VLProcessor`] uses - [`Qwen2VLImageProcessor`] for processing images. - pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): - The tensors corresponding to the input videos. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`Qwen2VLImageProcessor.__call__`] for details. [`Qwen2VLProcessor`] uses - [`Qwen2VLImageProcessor`] for processing videos. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): - The rope index difference between sequence length and multimodal rope. -""" - - +@auto_docstring class Qwen2VLModel(Qwen2VLPreTrainedModel): _checkpoint_conversion_mapping = {"^model": "language_model"} @@ -1608,7 +1507,7 @@ class Qwen2VLModel(Qwen2VLPreTrainedModel): return position_ids, mrope_position_deltas - @add_start_docstrings_to_model_forward(QWEN2_VL_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1627,6 +1526,19 @@ class Qwen2VLModel(Qwen2VLPreTrainedModel): rope_deltas: Optional[torch.LongTensor] = None, cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple, Qwen2VLModelOutputWithPast]: + r""" + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2VLImageProcessor.__call__`] for details. [`Qwen2VLProcessor`] uses + [`Qwen2VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1763,8 +1675,7 @@ class Qwen2VLForConditionalGeneration(Qwen2VLPreTrainedModel, GenerationMixin): return self.model.visual @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN2_VL_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Qwen2VLCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -1785,12 +1696,20 @@ class Qwen2VLForConditionalGeneration(Qwen2VLPreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple, Qwen2VLCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2VLImageProcessor.__call__`] for details. [`Qwen2VLProcessor`] uses + [`Qwen2VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. Example: diff --git a/src/transformers/models/qwen3/modeling_qwen3.py b/src/transformers/models/qwen3/modeling_qwen3.py index d44eb4fb9a..bc5a770a8c 100644 --- a/src/transformers/models/qwen3/modeling_qwen3.py +++ b/src/transformers/models/qwen3/modeling_qwen3.py @@ -41,16 +41,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_qwen3 import Qwen3Config @@ -62,9 +53,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Qwen/Qwen3-8B" -_CONFIG_FOR_DOC = "Qwen3Config" - @use_kernel_forward_from_hub("RMSNorm") class Qwen3RMSNorm(nn.Module): @@ -319,27 +307,7 @@ class Qwen3DecoderLayer(GradientCheckpointingLayer): return outputs -QWEN3_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen3Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen3 Model outputting raw hidden-states without any specific head on top.", - QWEN3_START_DOCSTRING, -) +@auto_docstring class Qwen3PreTrainedModel(PreTrainedModel): config_class = Qwen3Config base_model_prefix = "model" @@ -402,88 +370,8 @@ class Qwen3RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -QWEN3_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Qwen3 Model outputting raw hidden-states without any specific head on top.", - QWEN3_START_DOCSTRING, -) +@auto_docstring class Qwen3Model(Qwen3PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen3DecoderLayer`] - - Args: - config: Qwen3Config - """ - def __init__(self, config: Qwen3Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -507,7 +395,7 @@ class Qwen3Model(Qwen3PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -760,6 +648,7 @@ class Qwen3Model(Qwen3PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -793,8 +682,7 @@ class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -811,19 +699,10 @@ class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -878,8 +757,8 @@ class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Qwen3 Model transformer with a sequence classification head on top (linear layer). [`Qwen3ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -890,8 +769,7 @@ class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - QWEN3_START_DOCSTRING, + """ ) class Qwen3ForSequenceClassification(Qwen3PreTrainedModel): def __init__(self, config): @@ -910,7 +788,7 @@ class Qwen3ForSequenceClassification(Qwen3PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -979,13 +857,7 @@ class Qwen3ForSequenceClassification(Qwen3PreTrainedModel): ) -@add_start_docstrings( - """ - The Qwen3 Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - QWEN3_START_DOCSTRING, -) +@auto_docstring class Qwen3ForTokenClassification(Qwen3PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1010,12 +882,7 @@ class Qwen3ForTokenClassification(Qwen3PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1061,13 +928,7 @@ class Qwen3ForTokenClassification(Qwen3PreTrainedModel): ) -@add_start_docstrings( - """ -The Qwen3 Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - QWEN3_START_DOCSTRING, -) +@auto_docstring class Qwen3ForQuestionAnswering(Qwen3PreTrainedModel): base_model_prefix = "transformer" @@ -1086,7 +947,7 @@ class Qwen3ForQuestionAnswering(Qwen3PreTrainedModel): self.transformer.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1100,17 +961,6 @@ class Qwen3ForQuestionAnswering(Qwen3PreTrainedModel): output_hidden_states: Optional[bool] = None, **kwargs, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.transformer( input_ids, attention_mask=attention_mask, diff --git a/src/transformers/models/qwen3/modular_qwen3.py b/src/transformers/models/qwen3/modular_qwen3.py index 8e0b86da40..f914faa57f 100644 --- a/src/transformers/models/qwen3/modular_qwen3.py +++ b/src/transformers/models/qwen3/modular_qwen3.py @@ -24,10 +24,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - logging, -) +from ...utils import LossKwargs, logging from ..gemma.modeling_gemma import GemmaMLP from ..llama.modeling_llama import ( LlamaAttention, @@ -148,19 +145,10 @@ class Qwen3ForCausalLM(LlamaForCausalLM): **super_kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/qwen3_moe/configuration_qwen3_moe.py b/src/transformers/models/qwen3_moe/configuration_qwen3_moe.py index bac47bb85f..8b3219c9c3 100644 --- a/src/transformers/models/qwen3_moe/configuration_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/configuration_qwen3_moe.py @@ -26,7 +26,7 @@ class Qwen3MoeConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`Qwen3MoeModel`]. It is used to instantiate a Qwen3MoE model according to the specified arguments, defining the model architecture. Instantiating a configuration - with the defaults will yield a similar configuration to that of [Qwen/Qwen3-MoE-15B-A2B](https://huggingface.co/Qwen/Qwen3-15B-A2B). + with the defaults will yield a similar configuration to that of [Qwen/Qwen3-15B-A2B](https://huggingface.co/Qwen/Qwen3-15B-A2B). Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. diff --git a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py index 8c5c8f905b..6c2ac9e461 100644 --- a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -34,7 +34,6 @@ from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import ( BaseModelOutputWithPast, - CausalLMOutputWithPast, MoeCausalLMOutputWithPast, MoeModelOutputWithPast, QuestionAnsweringModelOutput, @@ -44,16 +43,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_qwen3_moe import Qwen3MoeConfig @@ -65,9 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Qwen/Qwen3-MoE-15B-A2B" -_CONFIG_FOR_DOC = "Qwen3MoeConfig" - def rotate_half(x): """Rotates half the hidden dims of the input.""" @@ -446,27 +433,7 @@ class Qwen3MoeRotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -QWEN3_MOE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Qwen3MoeConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Qwen3Moe Model outputting raw hidden-states without any specific head on top.", - QWEN3_MOE_START_DOCSTRING, -) +@auto_docstring class Qwen3MoePreTrainedModel(PreTrainedModel): config_class = Qwen3MoeConfig base_model_prefix = "model" @@ -495,88 +462,8 @@ class Qwen3MoePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -QWEN3_MOE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Qwen3Moe Model outputting raw hidden-states without any specific head on top.", - QWEN3_MOE_START_DOCSTRING, -) +@auto_docstring class Qwen3MoeModel(Qwen3MoePreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen3MoeDecoderLayer`] - - Args: - config: Qwen3MoeConfig - """ - def __init__(self, config: Qwen3MoeConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -600,7 +487,7 @@ class Qwen3MoeModel(Qwen3MoePreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_MOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -955,6 +842,7 @@ def load_balancing_loss_func( return overall_loss * num_experts +@auto_docstring class Qwen3MoeForCausalLM(Qwen3MoePreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -991,8 +879,7 @@ class Qwen3MoeForCausalLM(Qwen3MoePreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_MOE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1010,19 +897,10 @@ class Qwen3MoeForCausalLM(Qwen3MoePreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1096,8 +974,8 @@ class Qwen3MoeForCausalLM(Qwen3MoePreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Qwen3Moe Model transformer with a sequence classification head on top (linear layer). [`Qwen3MoeForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1108,8 +986,7 @@ class Qwen3MoeForCausalLM(Qwen3MoePreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - QWEN3_MOE_START_DOCSTRING, + """ ) class Qwen3MoeForSequenceClassification(Qwen3MoePreTrainedModel): def __init__(self, config): @@ -1128,7 +1005,7 @@ class Qwen3MoeForSequenceClassification(Qwen3MoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_MOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1197,13 +1074,7 @@ class Qwen3MoeForSequenceClassification(Qwen3MoePreTrainedModel): ) -@add_start_docstrings( - """ - The Qwen3Moe Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - QWEN3_MOE_START_DOCSTRING, -) +@auto_docstring class Qwen3MoeForTokenClassification(Qwen3MoePreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1228,12 +1099,7 @@ class Qwen3MoeForTokenClassification(Qwen3MoePreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_MOE_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1279,13 +1145,7 @@ class Qwen3MoeForTokenClassification(Qwen3MoePreTrainedModel): ) -@add_start_docstrings( - """ -The Qwen3Moe Model transformer with a span classification head on top for extractive question-answering tasks like -SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - QWEN3_MOE_START_DOCSTRING, -) +@auto_docstring class Qwen3MoeForQuestionAnswering(Qwen3MoePreTrainedModel): base_model_prefix = "transformer" @@ -1304,7 +1164,7 @@ class Qwen3MoeForQuestionAnswering(Qwen3MoePreTrainedModel): self.transformer.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(QWEN3_MOE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1318,17 +1178,6 @@ class Qwen3MoeForQuestionAnswering(Qwen3MoePreTrainedModel): output_hidden_states: Optional[bool] = None, **kwargs, ) -> QuestionAnsweringModelOutput: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ - outputs: BaseModelOutputWithPast = self.transformer( input_ids, attention_mask=attention_mask, diff --git a/src/transformers/models/qwen3_moe/modular_qwen3_moe.py b/src/transformers/models/qwen3_moe/modular_qwen3_moe.py index 92bea6ff60..3c673b6079 100644 --- a/src/transformers/models/qwen3_moe/modular_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modular_qwen3_moe.py @@ -25,21 +25,14 @@ from ...activations import ACT2FN from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - logging, -) +from ...utils import LossKwargs, logging from ..llama.modeling_llama import ( LlamaForQuestionAnswering, LlamaForSequenceClassification, LlamaForTokenClassification, LlamaRMSNorm, ) -from ..mixtral.modeling_mixtral import ( - MixtralForCausalLM, - MixtralModel, - load_balancing_loss_func, -) +from ..mixtral.modeling_mixtral import MixtralForCausalLM, MixtralModel, load_balancing_loss_func from ..qwen2_moe.modeling_qwen2_moe import Qwen2MoeDecoderLayer from ..qwen3.modeling_qwen3 import Qwen3Attention from .configuration_qwen3_moe import Qwen3MoeConfig @@ -47,8 +40,6 @@ from .configuration_qwen3_moe import Qwen3MoeConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "Qwen/Qwen3-MoE-15B-A2B" - class Qwen3MoeAttention(Qwen3Attention): # This is the main diff with qwen2Moe! pass @@ -256,19 +247,10 @@ class Qwen3MoeForCausalLM(MixtralForCausalLM): **kwargs: Unpack[KwargsForCausalLM], ) -> MoeCausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index e3b835492b..e2456afafa 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -25,15 +25,13 @@ from ...configuration_utils import PretrainedConfig from ...generation import GenerationConfig, GenerationMixin, LogitsProcessorList, StoppingCriteriaList from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_rag import RagConfig from .retrieval_rag import RagRetriever logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "RagConfig" - @dataclass class RetrievAugLMMarginOutput(ModelOutput): @@ -220,16 +218,16 @@ class RetrievAugLMOutput(ModelOutput): generator_cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None -class RagPreTrainedModel(PreTrainedModel): - r""" +@auto_docstring( + custom_intro=""" RAG models were released with the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/abs/2005.11401) by Patrick Lewis, Ethan Perez, Aleksandra Piktus et al. RAG is a retriever augmented model and encapsulate three components: a question encoder, a dataset retriever and a generator, the encoder and generator are trainable while the retriever is just an indexed dataset. - """ - +) +class RagPreTrainedModel(PreTrainedModel): config_class = RagConfig base_model_prefix = "rag" _supports_flash_attn_2 = True @@ -377,107 +375,7 @@ class RagPreTrainedModel(PreTrainedModel): return cls(question_encoder=question_encoder, generator=generator, config=config, retriever=retriever) -RAG_START_DOCSTRING = r""" - - RAG is a seq2seq model which encapsulates two core components: a question encoder and a generator. During a forward - pass, we encode the input with the question encoder and pass it to the retriever to extract relevant context - documents. The documents are then prepended to the input. Such contextualized inputs is passed to the generator. - - The question encoder can be any *autoencoding* model, preferably [`DPRQuestionEncoder`], and the generator can be - any *seq2seq* model, preferably [`BartForConditionalGeneration`]. - - The model can be initialized with a [`RagRetriever`] for end-to-end generation or used in combination with the - outputs of a retriever in multiple steps---see examples for more details. The model is compatible any - *autoencoding* model as the `question_encoder` and any *seq2seq* model with language model head as the `generator`. - It has been tested with [`DPRQuestionEncoder`] as the `question_encoder` and [`BartForConditionalGeneration`] or - [`T5ForConditionalGeneration`] as the `generator`. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - - Args: - config ([`RagConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. - question_encoder ([`PreTrainedModel`]): - An encoder model compatible with the faiss index encapsulated by the `retriever`. - generator ([`PreTrainedModel`]): - A seq2seq model used as the generator in the RAG architecture. - retriever ([`RagRetriever`]): - A retriever class encapsulating a faiss index queried to obtain context documents for current inputs. -""" - - -RAG_FORWARD_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize the model, specifies - which generator to use, it also specifies a compatible generator tokenizer. Use that tokenizer class to - obtain the indices. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*) - Tuple consists of (`generator_enc_last_hidden_state`, *optional*: `generator_enc_hidden_states`, - *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape `(batch_size, n_docs * - sequence_length, hidden_size)` is a sequence of hidden-states at the output of the last layer of the - generator's encoder. - - Used by the ([`RagModel`]) model during decoding. - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Provide for generation tasks. `None` by default, construct as per instructions for the generator model - you're using with your RAG instance. - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - past_key_values (`tuple(tuple(torch.FloatTensor))`): - Tuple consists of two elements: `encoder_outputs` of the RAG model (see `encoder_outputs`) and - `past_key_values` of the underlying generator. Can be used to speed up decoding. `past_key_values` are used - in the ([`RagTokenForGeneration`]) model during decoding. - doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`): - Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and - `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` `doc_scores` - has to be provided to the forward pass. `doc_scores` can be computed via - `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more information. - context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*): - Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the - retriever. If the model was not initialized with a `retriever` ``context_input_ids` has to be provided to - the forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. - context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`,*optional*, returned when *output_retrieved=True*): - Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the - retriever. If the model has is not initialized with a `retriever` `context_attention_mask` has to be - provided to the forward pass. `context_attention_mask` are returned by [`~RagRetriever.__call__`]. - use_cache (`bool`, *optional*, defaults to `True`): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_retrieved(`bool`, *optional*): - Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and - `context_attention_mask`. See returned tensors for more detail. - n_docs (`int`, *optional*, defaults to `config.n_docs``) - Number of documents to retrieve and/or number of documents for which to generate an answer. -""" - - -@add_start_docstrings_to_model_forward(RAG_START_DOCSTRING) +@auto_docstring class RagModel(RagPreTrainedModel): def __init__( self, @@ -487,6 +385,14 @@ class RagModel(RagPreTrainedModel): retriever: Optional[RagRetriever] = None, # or maybe just use a `set_retriever(...)` method **kwargs, ): + r""" + question_encoder (`PreTrainedModel`, *optional*): + The model responsible for encoding the question into hidden states for retrieval. + generator (`PreTrainedModel`, *optional*): + The model responsible for generating text based on retrieved documents. + retriever (`RagRetriever`, *optional*): + The component responsible for retrieving documents from a knowledge base given the encoded question. + """ assert config is not None or (question_encoder is not None and generator is not None), ( "Either a configuration or an question_encoder and a generator has to be provided." ) @@ -521,8 +427,7 @@ class RagModel(RagPreTrainedModel): self.ctx_encoder = None self.context_encoder_training = False - @add_start_docstrings_to_model_forward(RAG_FORWARD_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RetrievAugLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -541,7 +446,43 @@ class RagModel(RagPreTrainedModel): n_docs: Optional[int] = None, ) -> Union[Tuple[torch.Tensor], RetrievAugLMOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize the model, specifies + which generator to use, it also specifies a compatible generator tokenizer. Use that tokenizer class to + obtain the indices. + + [What are input IDs?](../glossary#input-ids) + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*) + Tuple consists of (`generator_enc_last_hidden_state`, *optional*: `generator_enc_hidden_states`, + *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape `(batch_size, n_docs * + sequence_length, hidden_size)` is a sequence of hidden-states at the output of the last layer of the + generator's encoder. + + Used by the ([`RagModel`]) model during decoding. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for generation tasks. `None` by default, construct as per instructions for the generator model + you're using with your RAG instance. + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`): + Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and + `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` `doc_scores` + has to be provided to the forward pass. `doc_scores` can be computed via + `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more information. + context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*): + Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the + retriever. If the model was not initialized with a `retriever` ``context_input_ids` has to be provided to + the forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. + context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`,*optional*, returned when *output_retrieved=True*): + Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the + retriever. If the model has is not initialized with a `retriever` `context_attention_mask` has to be + provided to the forward pass. `context_attention_mask` are returned by [`~RagRetriever.__call__`]. + output_retrieved (`bool`, *optional*): + Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and + `context_attention_mask`. See returned tensors for more detail. + n_docs (`int`, *optional*): + The number of documents to retrieve. Example: @@ -718,11 +659,10 @@ class RagModel(RagPreTrainedModel): ) -@add_start_docstrings_to_model_forward( - """ +@auto_docstring( + custom_intro=""" A RAG-sequence model implementation. It performs RAG-sequence specific marginalization in the forward pass. - """, - RAG_START_DOCSTRING, + """ ) class RagSequenceForGeneration(RagPreTrainedModel): def __init__( @@ -733,6 +673,14 @@ class RagSequenceForGeneration(RagPreTrainedModel): retriever: Optional[RagRetriever] = None, **kwargs, ): + r""" + question_encoder (`PreTrainedModel`, *optional*): + The model responsible for encoding the question into hidden states for retrieval. + generator (`PreTrainedModel`, *optional*): + The model responsible for generating text based on retrieved documents. + retriever (`RagRetriever`, *optional*): + The component responsible for retrieving documents from a knowledge base given the encoded question. + """ assert config is not None or (question_encoder is not None and generator is not None), ( "Either a configuration or an encoder and a generator has to be provided." ) @@ -753,8 +701,7 @@ class RagSequenceForGeneration(RagPreTrainedModel): self.rag.context_encoder_training = True self.rag.ctx_encoder = ctx_encoder - @add_start_docstrings_to_model_forward(RAG_FORWARD_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RetrievAugLMMarginOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -777,16 +724,49 @@ class RagSequenceForGeneration(RagPreTrainedModel): **kwargs, # needs kwargs for generation ) -> RetrievAugLMMarginOutput: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize the model, specifies + which generator to use, it also specifies a compatible generator tokenizer. Use that tokenizer class to + obtain the indices. + + [What are input IDs?](../glossary#input-ids) + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*) + Tuple consists of (`generator_enc_last_hidden_state`, *optional*: `generator_enc_hidden_states`, + *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape `(batch_size, n_docs * + sequence_length, hidden_size)` is a sequence of hidden-states at the output of the last layer of the + generator's encoder. + + Used by the ([`RagModel`]) model during decoding. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for generation tasks. `None` by default, construct as per instructions for the generator model + you're using with your RAG instance. + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*): + Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the + retriever. If the model was not initialized with a `retriever` ``context_input_ids` has to be provided to + the forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. + context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`,*optional*, returned when *output_retrieved=True*): + Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the + retriever. If the model has is not initialized with a `retriever` `context_attention_mask` has to be + provided to the forward pass. `context_attention_mask` are returned by [`~RagRetriever.__call__`]. + doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`): + Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and + `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` `doc_scores` + has to be provided to the forward pass. `doc_scores` can be computed via + `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more information. + output_retrieved (`bool`, *optional*): + Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and + `context_attention_mask`. See returned tensors for more detail. exclude_bos_score (`bool`, *optional*): Only relevant if `labels` is passed. If `True`, the score of the BOS token is disregarded when computing the loss. reduce_loss (`bool`, *optional*): Only relevant if `labels` is passed. If `True`, the NLL loss is reduced using the `torch.Tensor.sum` operation. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Legacy dictionary, which is required so that model can use *generate()* function. - - Returns: + n_docs (`int`, *optional*): + The number of documents to retrieve. Example: @@ -1116,11 +1096,10 @@ class RagSequenceForGeneration(RagPreTrainedModel): return output -@add_start_docstrings_to_model_forward( - """ +@auto_docstring( + custom_intro=""" A RAG-token model implementation. It performs RAG-token specific marginalization in the forward pass. - """, - RAG_START_DOCSTRING, + """ ) class RagTokenForGeneration(RagPreTrainedModel, GenerationMixin): def __init__( @@ -1131,6 +1110,14 @@ class RagTokenForGeneration(RagPreTrainedModel, GenerationMixin): retriever: Optional[RagRetriever] = None, **kwargs, ): + r""" + question_encoder (`PreTrainedModel`, *optional*): + The model responsible for encoding the question into hidden states for retrieval. + generator (`PreTrainedModel`, *optional*): + The model responsible for generating text based on retrieved documents. + retriever (`RagRetriever`, *optional*): + The component responsible for retrieving documents from a knowledge base given the encoded question. + """ assert config is not None or (question_encoder is not None and generator is not None), ( "Either a configuration or an encoder and a generator has to be provided." ) @@ -1224,8 +1211,7 @@ class RagTokenForGeneration(RagPreTrainedModel, GenerationMixin): log_prob_sum = seq_logprobs + doc_logprobs.unsqueeze(-1).unsqueeze(-1) return torch.logsumexp(log_prob_sum, dim=1) - @add_start_docstrings_to_model_forward(RAG_FORWARD_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RetrievAugLMMarginOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1248,16 +1234,49 @@ class RagTokenForGeneration(RagPreTrainedModel, GenerationMixin): **kwargs, # needs kwargs for generation ) -> RetrievAugLMMarginOutput: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize the model, specifies + which generator to use, it also specifies a compatible generator tokenizer. Use that tokenizer class to + obtain the indices. + + [What are input IDs?](../glossary#input-ids) + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*) + Tuple consists of (`generator_enc_last_hidden_state`, *optional*: `generator_enc_hidden_states`, + *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape `(batch_size, n_docs * + sequence_length, hidden_size)` is a sequence of hidden-states at the output of the last layer of the + generator's encoder. + + Used by the ([`RagModel`]) model during decoding. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Provide for generation tasks. `None` by default, construct as per instructions for the generator model + you're using with your RAG instance. + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*): + Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the + retriever. If the model was not initialized with a `retriever` ``context_input_ids` has to be provided to + the forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. + context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`,*optional*, returned when *output_retrieved=True*): + Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the + retriever. If the model has is not initialized with a `retriever` `context_attention_mask` has to be + provided to the forward pass. `context_attention_mask` are returned by [`~RagRetriever.__call__`]. + doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`): + Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and + `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` `doc_scores` + has to be provided to the forward pass. `doc_scores` can be computed via + `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more information. + output_retrieved (`bool`, *optional*): + Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and + `context_attention_mask`. See returned tensors for more detail. do_marginalize (`bool`, *optional*): If `True`, the logits are marginalized over all documents by making use of `torch.nn.functional.log_softmax`. reduce_loss (`bool`, *optional*): Only relevant if `labels` is passed. If `True`, the NLL loss is reduced using the `torch.Tensor.sum` operation. - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Legacy dictionary, which is required so that model can use *generate()* function. - - Returns: + n_docs (`int`, *optional*): + The number of documents to retrieve. Example: diff --git a/src/transformers/models/recurrent_gemma/modeling_recurrent_gemma.py b/src/transformers/models/recurrent_gemma/modeling_recurrent_gemma.py index fcf2402ec9..315edd2986 100644 --- a/src/transformers/models/recurrent_gemma/modeling_recurrent_gemma.py +++ b/src/transformers/models/recurrent_gemma/modeling_recurrent_gemma.py @@ -28,18 +28,12 @@ from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_outputs import BaseModelOutputWithNoAttention, CausalLMOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.import_utils import is_torchdynamo_compiling from .configuration_recurrent_gemma import RecurrentGemmaConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "RecurrentGemmaConfig" _MAX_SQRT_GRADIENT = 1000.0 @@ -511,27 +505,7 @@ class RecurrentGemmaDecoderLayer(nn.Module): return hidden_states -RECURRENTGEMMA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`RecurrentGemmaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare RecurrentGemma Model outputting raw hidden-states without any specific head on top.", - RECURRENTGEMMA_START_DOCSTRING, -) +@auto_docstring class RecurrentGemmaPreTrainedModel(PreTrainedModel): config_class = RecurrentGemmaConfig base_model_prefix = "model" @@ -598,63 +572,8 @@ class RecurrentGemmaPreTrainedModel(PreTrainedModel): pass -RECURRENTGEMMA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare RecurrentGemma Model outputting raw hidden-states without any specific head on top.", - RECURRENTGEMMA_START_DOCSTRING, -) +@auto_docstring class RecurrentGemmaModel(RecurrentGemmaPreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`RecurrentGemmaDecoderLayer`] - - Args: - config: RecurrentGemmaConfig - """ - def __init__(self, config: RecurrentGemmaConfig): super().__init__(config) self.padding_idx = config.pad_token_id @@ -681,7 +600,7 @@ class RecurrentGemmaModel(RecurrentGemmaPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(RECURRENTGEMMA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -783,6 +702,7 @@ class RecurrentGemmaModel(RecurrentGemmaPreTrainedModel): # TODO: re-enable check: Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM with LLAMA->RECURRENTGEMMA,Llama->RecurrentGemma,llama->gemma +@auto_docstring class RecurrentGemmaForCausalLM(RecurrentGemmaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] @@ -813,9 +733,8 @@ class RecurrentGemmaForCausalLM(RecurrentGemmaPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model + @auto_docstring # Ignore copy - @add_start_docstrings_to_model_forward(RECURRENTGEMMA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutput, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -830,12 +749,10 @@ class RecurrentGemmaForCausalLM(RecurrentGemmaPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutput]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/reformer/modeling_reformer.py b/src/transformers/models/reformer/modeling_reformer.py index 48be78e7d4..3422fbad2b 100755 --- a/src/transformers/models/reformer/modeling_reformer.py +++ b/src/transformers/models/reformer/modeling_reformer.py @@ -37,20 +37,14 @@ from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_reformer import ReformerConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/reformer-crime-and-punishment" -_CONFIG_FOR_DOC = "ReformerConfig" - # Define named tuples for nn.Modules here LSHSelfAttentionOutput = namedtuple("LSHSelfAttentionOutput", ["hidden_states", "attention_probs", "buckets"]) @@ -1775,12 +1769,8 @@ class ReformerOnlyLMHead(nn.Module): self.bias = self.decoder.bias +@auto_docstring class ReformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ReformerConfig base_model_prefix = "reformer" @@ -1891,87 +1881,7 @@ class ReformerModelWithLMHeadOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -REFORMER_START_DOCSTRING = r""" - Reformer was proposed in [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, - Łukasz Kaiser, Anselm Levskaya. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ReformerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -REFORMER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. During training the input_ids sequence_length has to be - a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices - are automatically padded to be a multiple of the chunk length. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - num_hashes (`int`, *optional*): - The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites - the default defined in `config.num_hashes`. - - For more information, see `num_hashes` in [`ReformerConfig`]. - past_buckets_states (`List[Tuple(torch.LongTensor, torch.FloatTensor)]`, *optional*): - List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first element - being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`) and the - second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`). - - Contains precomputed hidden-states and buckets (only relevant for LSH Self-Attention). Can be used to speed - up sequential decoding. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Reformer Model transformer outputting raw hidden-stateswithout any specific head on top.", - REFORMER_START_DOCSTRING, -) +@auto_docstring class ReformerModel(ReformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2000,12 +1910,7 @@ class ReformerModel(ReformerPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(REFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=ReformerModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2020,6 +1925,29 @@ class ReformerModel(ReformerPreTrainedModel): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, ReformerModelOutput]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. During training the input_ids sequence_length has to be + a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices + are automatically padded to be a multiple of the chunk length. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + num_hashes (`int`, *optional*): + The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites + the default defined in `config.num_hashes`. + + For more information, see `num_hashes` in [`ReformerConfig`]. + past_buckets_states (`List[Tuple(torch.LongTensor, torch.FloatTensor)]`, *optional*): + List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first element + being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`) and the + second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`). + + Contains precomputed hidden-states and buckets (only relevant for LSH Self-Attention). Can be used to speed + up sequential decoding. + """ use_cache = use_cache if use_cache is not None else self.config.use_cache output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -2183,7 +2111,11 @@ class ReformerModel(ReformerPreTrainedModel): return input_ids, inputs_embeds, attention_mask, position_ids, input_shape -@add_start_docstrings("""Reformer Model with a `language modeling` head on top.""", REFORMER_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + Reformer Model with a `language modeling` head on top. + """ +) class ReformerModelWithLMHead(ReformerPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -2212,12 +2144,7 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel, GenerationMixin): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(REFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2235,10 +2162,31 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. During training the input_ids sequence_length has to be + a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices + are automatically padded to be a multiple of the chunk length. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + num_hashes (`int`, *optional*): + The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites + the default defined in `config.num_hashes`. + + For more information, see `num_hashes` in [`ReformerConfig`]. + past_buckets_states (`List[Tuple(torch.LongTensor, torch.FloatTensor)]`, *optional*): + List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first element + being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`) and the + second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`). + + Contains precomputed hidden-states and buckets (only relevant for LSH Self-Attention). Can be used to speed + up sequential decoding. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., - config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for - labels in `[0, ..., config.vocab_size]` + Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., + config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for + labels in `[0, ..., config.vocab_size]` """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2313,7 +2261,7 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel, GenerationMixin): return reord_past_buckets_states -@add_start_docstrings("""Reformer Model with a `language modeling` head on top.""", REFORMER_START_DOCSTRING) +@auto_docstring class ReformerForMaskedLM(ReformerPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -2336,8 +2284,7 @@ class ReformerForMaskedLM(ReformerPreTrainedModel): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(REFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2352,19 +2299,31 @@ class ReformerForMaskedLM(ReformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MaskedLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. During training the input_ids sequence_length has to be + a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices + are automatically padded to be a multiple of the chunk length. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + num_hashes (`int`, *optional*): + The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites + the default defined in `config.num_hashes`. + + For more information, see `num_hashes` in [`ReformerConfig`]. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), - the loss is only computed for the tokens with labels + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels - Returns: + - + This example uses a false checkpoint since we don't have any available pretrained model for the masked language + modeling task with the Reformer architecture. - This example uses a false checkpoint since we don't have any available pretrained model for the masked language - modeling task with the Reformer architecture. - - + Example: @@ -2438,12 +2397,11 @@ class ReformerForMaskedLM(ReformerPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Reformer Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - REFORMER_START_DOCSTRING, + """ ) class ReformerForSequenceClassification(ReformerPreTrainedModel): def __init__(self, config): @@ -2459,8 +2417,7 @@ class ReformerForSequenceClassification(ReformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2475,13 +2432,25 @@ class ReformerForSequenceClassification(ReformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. During training the input_ids sequence_length has to be + a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices + are automatically padded to be a multiple of the chunk length. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + num_hashes (`int`, *optional*): + The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites + the default defined in `config.num_hashes`. + + For more information, see `num_hashes` in [`ReformerConfig`]. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Example of single-label classification: ```python @@ -2585,13 +2554,7 @@ class ReformerClassificationHead(nn.Module): return hidden_states -@add_start_docstrings( - """ - Reformer Model with a span classification head on top for extractive question-answering tasks like SQuAD / TriviaQA - ( a linear layer on top of hidden-states output to compute `span start logits` and `span end logits`. - """, - REFORMER_START_DOCSTRING, -) +@auto_docstring class ReformerForQuestionAnswering(ReformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2604,12 +2567,7 @@ class ReformerForQuestionAnswering(ReformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2625,14 +2583,20 @@ class ReformerForQuestionAnswering(ReformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. During training the input_ids sequence_length has to be + a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices + are automatically padded to be a multiple of the chunk length. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + num_hashes (`int`, *optional*): + The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites + the default defined in `config.num_hashes`. + + For more information, see `num_hashes` in [`ReformerConfig`]. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/regnet/modeling_regnet.py b/src/transformers/models/regnet/modeling_regnet.py index 9fd0a4c634..b510e3565a 100644 --- a/src/transformers/models/regnet/modeling_regnet.py +++ b/src/transformers/models/regnet/modeling_regnet.py @@ -23,30 +23,18 @@ from torch import Tensor, nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward from ...modeling_outputs import ( BaseModelOutputWithNoAttention, BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import logging +from ...utils import auto_docstring, logging from .configuration_regnet import RegNetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "RegNetConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/regnet-y-040" -_EXPECTED_OUTPUT_SHAPE = [1, 1088, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "facebook/regnet-y-040" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - class RegNetConvLayer(nn.Module): def __init__( @@ -270,12 +258,8 @@ class RegNetEncoder(nn.Module): return BaseModelOutputWithNoAttention(last_hidden_state=hidden_state, hidden_states=hidden_states) +@auto_docstring class RegNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RegNetConfig base_model_prefix = "regnet" main_input_name = "pixel_values" @@ -297,35 +281,7 @@ class RegNetPreTrainedModel(PreTrainedModel): nn.init.constant_(module.bias, 0) -REGNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matters related to general usage and - behavior. - - Parameters: - config ([`RegNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -REGNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`ConvNextImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RegNet model outputting raw features without any specific head on top.", - REGNET_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet class RegNetModel(RegNetPreTrainedModel): def __init__(self, config): @@ -337,14 +293,7 @@ class RegNetModel(RegNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REGNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BaseModelOutputWithPoolingAndNoAttention: @@ -373,12 +322,11 @@ class RegNetModel(RegNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - REGNET_START_DOCSTRING, + """ ) # Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet class RegNetForImageClassification(RegNetPreTrainedModel): @@ -394,13 +342,7 @@ class RegNetForImageClassification(RegNetPreTrainedModel): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REGNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, diff --git a/src/transformers/models/rembert/modeling_rembert.py b/src/transformers/models/rembert/modeling_rembert.py index ac4ee0a970..6a8ba4bf11 100755 --- a/src/transformers/models/rembert/modeling_rembert.py +++ b/src/transformers/models/rembert/modeling_rembert.py @@ -37,21 +37,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_rembert import RemBertConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "RemBertConfig" -_CHECKPOINT_FOR_DOC = "google/rembert" - def load_tf_weights_in_rembert(model, config, tf_checkpoint_path): """Load tf checkpoints in a pytorch model.""" @@ -636,12 +627,8 @@ class RemBertOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class RemBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RemBertConfig load_tf_weights = load_tf_weights_in_rembert base_model_prefix = "rembert" @@ -664,74 +651,8 @@ class RemBertPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -REMBERT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`RemBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -REMBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RemBERT Model transformer outputting raw hidden-states without any specific head on top.", - REMBERT_START_DOCSTRING, -) -class RemBertModel(RemBertPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, @@ -741,8 +662,13 @@ class RemBertModel(RemBertPreTrainedModel): to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. """ - +) +class RemBertModel(RemBertPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -768,12 +694,7 @@ class RemBertModel(RemBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/rembert", - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -790,25 +711,6 @@ class RemBertModel(RemBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -898,7 +800,7 @@ class RemBertModel(RemBertPreTrainedModel): ) -@add_start_docstrings("""RemBERT Model with a `language modeling` head on top.""", REMBERT_START_DOCSTRING) +@auto_docstring class RemBertForMaskedLM(RemBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight"] @@ -923,12 +825,7 @@ class RemBertForMaskedLM(RemBertPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.cls.predictions.decoder = new_embeddings - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/rembert", - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1008,8 +905,10 @@ class RemBertForMaskedLM(RemBertPreTrainedModel): return False -@add_start_docstrings( - """RemBERT Model with a `language modeling` head on top for CLM fine-tuning.""", REMBERT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + RemBERT Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class RemBertForCausalLM(RemBertPreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder.weight"] @@ -1032,8 +931,7 @@ class RemBertForCausalLM(RemBertPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.cls.predictions.decoder = new_embeddings - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1053,29 +951,10 @@ class RemBertForCausalLM(RemBertPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1146,12 +1025,11 @@ class RemBertForCausalLM(RemBertPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - REMBERT_START_DOCSTRING, + """ ) class RemBertForSequenceClassification(RemBertPreTrainedModel): def __init__(self, config): @@ -1164,12 +1042,7 @@ class RemBertForSequenceClassification(RemBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/rembert", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.FloatTensor] = None, @@ -1242,13 +1115,7 @@ class RemBertForSequenceClassification(RemBertPreTrainedModel): ) -@add_start_docstrings( - """ - RemBERT Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - REMBERT_START_DOCSTRING, -) +@auto_docstring class RemBertForMultipleChoice(RemBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1260,12 +1127,7 @@ class RemBertForMultipleChoice(RemBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/rembert", - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.FloatTensor] = None, @@ -1280,6 +1142,30 @@ class RemBertForMultipleChoice(RemBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1333,13 +1219,7 @@ class RemBertForMultipleChoice(RemBertPreTrainedModel): ) -@add_start_docstrings( - """ - RemBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - REMBERT_START_DOCSTRING, -) +@auto_docstring class RemBertForTokenClassification(RemBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1352,12 +1232,7 @@ class RemBertForTokenClassification(RemBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/rembert", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.FloatTensor] = None, @@ -1411,13 +1286,7 @@ class RemBertForTokenClassification(RemBertPreTrainedModel): ) -@add_start_docstrings( - """ - RemBERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - REMBERT_START_DOCSTRING, -) +@auto_docstring class RemBertForQuestionAnswering(RemBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1430,12 +1299,7 @@ class RemBertForQuestionAnswering(RemBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(REMBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="google/rembert", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.FloatTensor] = None, @@ -1450,16 +1314,6 @@ class RemBertForQuestionAnswering(RemBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.rembert( diff --git a/src/transformers/models/resnet/modeling_resnet.py b/src/transformers/models/resnet/modeling_resnet.py index d3b7574148..c913bb1bdd 100644 --- a/src/transformers/models/resnet/modeling_resnet.py +++ b/src/transformers/models/resnet/modeling_resnet.py @@ -30,30 +30,13 @@ from ...modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_resnet import ResNetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "ResNetConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/resnet-50" -_EXPECTED_OUTPUT_SHAPE = [1, 2048, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "microsoft/resnet-50" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tiger cat" - class ResNetConvLayer(nn.Module): def __init__( @@ -261,12 +244,8 @@ class ResNetEncoder(nn.Module): ) +@auto_docstring class ResNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ResNetConfig base_model_prefix = "resnet" main_input_name = "pixel_values" @@ -287,35 +266,7 @@ class ResNetPreTrainedModel(PreTrainedModel): nn.init.constant_(module.bias, 0) -RESNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ResNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -RESNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`ConvNextImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ResNet model outputting raw features without any specific head on top.", - RESNET_START_DOCSTRING, -) +@auto_docstring class ResNetModel(ResNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -326,14 +277,7 @@ class ResNetModel(ResNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(RESNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BaseModelOutputWithPoolingAndNoAttention: @@ -362,12 +306,11 @@ class ResNetModel(ResNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - RESNET_START_DOCSTRING, + """ ) class ResNetForImageClassification(ResNetPreTrainedModel): def __init__(self, config): @@ -382,13 +325,7 @@ class ResNetForImageClassification(ResNetPreTrainedModel): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(RESNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -439,11 +376,10 @@ class ResNetForImageClassification(ResNetPreTrainedModel): return ImageClassifierOutputWithNoAttention(loss=loss, logits=logits, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ResNet backbone, to be used with frameworks like DETR and MaskFormer. - """, - RESNET_START_DOCSTRING, + """ ) class ResNetBackbone(ResNetPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -457,14 +393,11 @@ class ResNetBackbone(ResNetPreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(RESNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/roberta/modeling_roberta.py b/src/transformers/models/roberta/modeling_roberta.py index 8b77f2bd63..a88cf0c9c6 100644 --- a/src/transformers/models/roberta/modeling_roberta.py +++ b/src/transformers/models/roberta/modeling_roberta.py @@ -26,10 +26,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN, gelu from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - _prepare_4d_attention_mask_for_sdpa, - _prepare_4d_causal_attention_mask_for_sdpa, -) +from ...modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -42,22 +39,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - get_torch_version, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, get_torch_version, logging from .configuration_roberta import RobertaConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "FacebookAI/roberta-base" -_CONFIG_FOR_DOC = "RobertaConfig" - class RobertaEmbeddings(nn.Module): """ @@ -686,12 +673,8 @@ class RobertaPooler(nn.Module): return pooled_output +@auto_docstring class RobertaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RobertaConfig base_model_prefix = "roberta" supports_gradient_checkpointing = True @@ -718,81 +701,8 @@ class RobertaPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -ROBERTA_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`RobertaConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ROBERTA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value - >= 2. All the value in this tensor should be always < type_vocab_size. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.", - ROBERTA_START_DOCSTRING, -) -# Copied from transformers.models.bert.modeling_bert.BertModel with Bert->Roberta, BERT->ROBERTA -class RobertaModel(RobertaPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, @@ -802,10 +712,16 @@ class RobertaModel(RobertaPreTrainedModel): to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. """ - +) +# Copied from transformers.models.bert.modeling_bert.BertModel with Bert->Roberta, BERT->ROBERTA +class RobertaModel(RobertaPreTrainedModel): _no_split_modules = ["RobertaEmbeddings", "RobertaLayer"] def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -834,12 +750,7 @@ class RobertaModel(RobertaPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -856,26 +767,6 @@ class RobertaModel(RobertaPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, target_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1003,8 +894,10 @@ class RobertaModel(RobertaPreTrainedModel): ) -@add_start_docstrings( - """RoBERTa Model with a `language modeling` head on top for CLM fine-tuning.""", ROBERTA_START_DOCSTRING +@auto_docstring( + custom_intro=""" + RoBERTa Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class RobertaForCausalLM(RobertaPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1027,8 +920,7 @@ class RobertaForCausalLM(RobertaPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1048,31 +940,19 @@ class RobertaForCausalLM(RobertaPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1146,7 +1026,7 @@ class RobertaForCausalLM(RobertaPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top.""", ROBERTA_START_DOCSTRING) +@auto_docstring class RobertaForMaskedLM(RobertaPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1171,15 +1051,7 @@ class RobertaForMaskedLM(RobertaPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - expected_output="' Paris'", - expected_loss=0.1, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1196,12 +1068,19 @@ class RobertaForMaskedLM(RobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1271,12 +1150,11 @@ class RobertaLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ROBERTA_START_DOCSTRING, + """ ) class RobertaForSequenceClassification(RobertaPreTrainedModel): def __init__(self, config): @@ -1290,14 +1168,7 @@ class RobertaForSequenceClassification(RobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="cardiffnlp/twitter-roberta-base-emotion", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'optimism'", - expected_loss=0.08, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1312,6 +1183,15 @@ class RobertaForSequenceClassification(RobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1370,13 +1250,7 @@ class RobertaForSequenceClassification(RobertaPreTrainedModel): ) -@add_start_docstrings( - """ - Roberta Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - ROBERTA_START_DOCSTRING, -) +@auto_docstring class RobertaForMultipleChoice(RobertaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1388,12 +1262,7 @@ class RobertaForMultipleChoice(RobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1408,10 +1277,35 @@ class RobertaForMultipleChoice(RobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1462,13 +1356,7 @@ class RobertaForMultipleChoice(RobertaPreTrainedModel): ) -@add_start_docstrings( - """ - Roberta Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - ROBERTA_START_DOCSTRING, -) +@auto_docstring class RobertaForTokenClassification(RobertaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1484,14 +1372,7 @@ class RobertaForTokenClassification(RobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="Jean-Baptiste/roberta-large-ner-english", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="['O', 'ORG', 'ORG', 'O', 'O', 'O', 'O', 'O', 'LOC', 'O', 'LOC', 'LOC']", - expected_loss=0.01, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1506,6 +1387,15 @@ class RobertaForTokenClassification(RobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1569,13 +1459,7 @@ class RobertaClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - Roberta Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - ROBERTA_START_DOCSTRING, -) +@auto_docstring class RobertaForQuestionAnswering(RobertaPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1587,14 +1471,7 @@ class RobertaForQuestionAnswering(RobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="deepset/roberta-base-squad2", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="' puppet'", - expected_loss=0.86, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1610,14 +1487,15 @@ class RobertaForQuestionAnswering(RobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py b/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py index 35c6550ff9..0f3aeaf4d0 100644 --- a/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py +++ b/src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py @@ -37,21 +37,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_roberta_prelayernorm import RobertaPreLayerNormConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "andreasmadsen/efficient_mlm_m0.40" -_CONFIG_FOR_DOC = "RobertaPreLayerNormConfig" - # Copied from transformers.models.roberta.modeling_roberta.RobertaEmbeddings with Roberta->RobertaPreLayerNorm class RobertaPreLayerNormEmbeddings(nn.Module): @@ -570,12 +561,8 @@ class RobertaPreLayerNormPooler(nn.Module): return pooled_output +@auto_docstring class RobertaPreLayerNormPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RobertaPreLayerNormConfig base_model_prefix = "roberta_prelayernorm" supports_gradient_checkpointing = True @@ -601,79 +588,8 @@ class RobertaPreLayerNormPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -ROBERTA_PRELAYERNORM_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`RobertaPreLayerNormConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value - >= 2. All the value in this tensor should be always < type_vocab_size. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RoBERTa-PreLayerNorm Model transformer outputting raw hidden-states without any specific head on top.", - ROBERTA_PRELAYERNORM_START_DOCSTRING, -) -class RobertaPreLayerNormModel(RobertaPreLayerNormPreTrainedModel): - """ +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is @@ -685,10 +601,14 @@ class RobertaPreLayerNormModel(RobertaPreLayerNormPreTrainedModel): `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 - """ - +) +class RobertaPreLayerNormModel(RobertaPreLayerNormPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -715,12 +635,7 @@ class RobertaPreLayerNormModel(RobertaPreLayerNormPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -738,24 +653,15 @@ class RobertaPreLayerNormModel(RobertaPreLayerNormPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). + [What are token type IDs?](../glossary#token-type-ids) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -853,9 +759,10 @@ class RobertaPreLayerNormModel(RobertaPreLayerNormPreTrainedModel): ) -@add_start_docstrings( - """RoBERTa-PreLayerNorm Model with a `language modeling` head on top for CLM fine-tuning.""", - ROBERTA_PRELAYERNORM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + RoBERTa-PreLayerNorm Model with a `language modeling` head on top for CLM fine-tuning. + """ ) # Copied from transformers.models.roberta.modeling_roberta.RobertaForCausalLM with FacebookAI/roberta-base->andreasmadsen/efficient_mlm_m0.40,ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta->roberta_prelayernorm, RobertaPreLayerNormTokenizer->RobertaTokenizer class RobertaPreLayerNormForCausalLM(RobertaPreLayerNormPreTrainedModel, GenerationMixin): @@ -881,8 +788,7 @@ class RobertaPreLayerNormForCausalLM(RobertaPreLayerNormPreTrainedModel, Generat def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -902,31 +808,19 @@ class RobertaPreLayerNormForCausalLM(RobertaPreLayerNormPreTrainedModel, Generat **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1000,8 +894,10 @@ class RobertaPreLayerNormForCausalLM(RobertaPreLayerNormPreTrainedModel, Generat return reordered_past -@add_start_docstrings( - """RoBERTa-PreLayerNorm Model with a `language modeling` head on top.""", ROBERTA_PRELAYERNORM_START_DOCSTRING +@auto_docstring( + custom_intro=""" + RoBERTa-PreLayerNorm Model with a `language modeling` head on top. + """ ) class RobertaPreLayerNormForMaskedLM(RobertaPreLayerNormPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1028,15 +924,7 @@ class RobertaPreLayerNormForMaskedLM(RobertaPreLayerNormPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - expected_output="' Paris'", - expected_loss=0.69, - ) + @auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForMaskedLM.forward with ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta->roberta_prelayernorm def forward( self, @@ -1054,12 +942,19 @@ class RobertaPreLayerNormForMaskedLM(RobertaPreLayerNormPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1130,12 +1025,11 @@ class RobertaPreLayerNormLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RoBERTa-PreLayerNorm Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ROBERTA_PRELAYERNORM_START_DOCSTRING, + """ ) class RobertaPreLayerNormForSequenceClassification(RobertaPreLayerNormPreTrainedModel): def __init__(self, config): @@ -1149,12 +1043,7 @@ class RobertaPreLayerNormForSequenceClassification(RobertaPreLayerNormPreTrained # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification.forward with roberta->roberta_prelayernorm def forward( self, @@ -1170,6 +1059,15 @@ class RobertaPreLayerNormForSequenceClassification(RobertaPreLayerNormPreTrained return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1228,13 +1126,7 @@ class RobertaPreLayerNormForSequenceClassification(RobertaPreLayerNormPreTrained ) -@add_start_docstrings( - """ - RobertaPreLayerNorm Model with a multiple choice classification head on top (a linear layer on top of the pooled - output and a softmax) e.g. for RocStories/SWAG tasks. - """, - ROBERTA_PRELAYERNORM_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForMultipleChoice with ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta->roberta_prelayernorm class RobertaPreLayerNormForMultipleChoice(RobertaPreLayerNormPreTrainedModel): def __init__(self, config): @@ -1247,14 +1139,7 @@ class RobertaPreLayerNormForMultipleChoice(RobertaPreLayerNormPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1269,10 +1154,35 @@ class RobertaPreLayerNormForMultipleChoice(RobertaPreLayerNormPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1323,13 +1233,7 @@ class RobertaPreLayerNormForMultipleChoice(RobertaPreLayerNormPreTrainedModel): ) -@add_start_docstrings( - """ - RobertaPreLayerNorm Model with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - ROBERTA_PRELAYERNORM_START_DOCSTRING, -) +@auto_docstring class RobertaPreLayerNormForTokenClassification(RobertaPreLayerNormPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1345,12 +1249,7 @@ class RobertaPreLayerNormForTokenClassification(RobertaPreLayerNormPreTrainedMod # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForTokenClassification.forward with roberta->roberta_prelayernorm def forward( self, @@ -1366,6 +1265,15 @@ class RobertaPreLayerNormForTokenClassification(RobertaPreLayerNormPreTrainedMod return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1430,13 +1338,7 @@ class RobertaPreLayerNormClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - RobertaPreLayerNorm Model with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - ROBERTA_PRELAYERNORM_START_DOCSTRING, -) +@auto_docstring class RobertaPreLayerNormForQuestionAnswering(RobertaPreLayerNormPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1448,12 +1350,7 @@ class RobertaPreLayerNormForQuestionAnswering(RobertaPreLayerNormPreTrainedModel # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROBERTA_PRELAYERNORM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForQuestionAnswering.forward with roberta->roberta_prelayernorm def forward( self, @@ -1470,14 +1367,15 @@ class RobertaPreLayerNormForQuestionAnswering(RobertaPreLayerNormPreTrainedModel return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/roc_bert/modeling_roc_bert.py b/src/transformers/models/roc_bert/modeling_roc_bert.py index b5ca264fb7..42e9fb94e4 100644 --- a/src/transformers/models/roc_bert/modeling_roc_bert.py +++ b/src/transformers/models/roc_bert/modeling_roc_bert.py @@ -37,43 +37,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_roc_bert import RoCBertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "weiweishi/roc-bert-base-zh" -_CONFIG_FOR_DOC = "RoCBertConfig" - -# Base model docstring -_EXPECTED_OUTPUT_SHAPE = [1, 8, 768] - -# Token Classification output -_CHECKPOINT_FOR_TOKEN_CLASSIFICATION = "ArthurZ/dummy-rocbert-ner" -_TOKEN_CLASS_EXPECTED_OUTPUT = ["S-EVENT", "S-FAC", "I-ORDINAL", "I-ORDINAL", "E-ORG", "E-LANGUAGE", "E-ORG", "E-ORG", "E-ORG", "E-ORG", "I-EVENT", "S-TIME", "S-TIME", "E-LANGUAGE", "S-TIME", "E-DATE", "I-ORDINAL", "E-QUANTITY", "E-LANGUAGE", "S-TIME", "B-ORDINAL", "S-PRODUCT", "E-LANGUAGE", "E-LANGUAGE", "E-ORG", "E-LOC", "S-TIME", "I-ORDINAL", "S-FAC", "O", "S-GPE", "I-EVENT", "S-GPE", "E-LANGUAGE", "E-ORG", "S-EVENT", "S-FAC", "S-FAC", "S-FAC", "E-ORG", "S-FAC", "E-ORG", "S-GPE"] # fmt: skip -_TOKEN_CLASS_EXPECTED_LOSS = 3.62 - -# SequenceClassification docstring -_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "ArthurZ/dummy-rocbert-seq" -_SEQ_CLASS_EXPECTED_OUTPUT = "'financial news'" -_SEQ_CLASS_EXPECTED_LOSS = 2.31 - -# QuestionAsnwering docstring -_CHECKPOINT_FOR_QA = "ArthurZ/dummy-rocbert-qa" -_QA_EXPECTED_OUTPUT = "''" -_QA_EXPECTED_LOSS = 3.75 -_QA_TARGET_START_INDEX = 14 -_QA_TARGET_END_INDEX = 15 - -# Maske language modeling - # Copied from transformers.models.bert.modeling_bert.load_tf_weights_in_bert with bert->roc_bert def load_tf_weights_in_roc_bert(model, config, tf_checkpoint_path): @@ -768,12 +737,8 @@ class RoCBertOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class RoCBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RoCBertConfig load_tf_weights = load_tf_weights_in_roc_bert base_model_prefix = "roc_bert" @@ -798,87 +763,8 @@ class RoCBertPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -ROC_BERT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`RoCBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ROC_BERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_shape_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the shape vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input_shape_ids) - input_pronunciation_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the pronunciation vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input_pronunciation_ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RoCBert Model transformer outputting raw hidden-states without any specific head on top.", - ROC_BERT_START_DOCSTRING, -) -class RoCBertModel(RoCBertPreTrainedModel): - """ +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is @@ -889,9 +775,14 @@ class RoCBertModel(RoCBertPreTrainedModel): to `True`. To be used in a Seq2Seq model, the model needs to be initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. """ - +) +class RoCBertModel(RoCBertPreTrainedModel): # Copied from transformers.models.clap.modeling_clap.ClapTextModel.__init__ with ClapText->RoCBert def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -932,13 +823,7 @@ class RoCBertModel(RoCBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -958,23 +843,20 @@ class RoCBertModel(RoCBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1073,11 +955,10 @@ class RoCBertModel(RoCBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RoCBert Model with contrastive loss and masked_lm_loss during the pretraining. - """, - ROC_BERT_START_DOCSTRING, + """ ) class RoCBertForPreTraining(RoCBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -1100,8 +981,7 @@ class RoCBertForPreTraining(RoCBertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1128,36 +1008,57 @@ class RoCBertForPreTraining(RoCBertPreTrainedModel): **kwargs, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" - attack_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - attack sample ids for computing the contrastive loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), - the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - attack_input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - attack sample shape ids for computing the contrastive loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), - the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - attack_input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - attack sample pronunciation ids for computing the contrastive loss. Indices should be in `[-100, 0, - ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - labels_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - target ids for computing the contrastive loss and masked_lm_loss . Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), - the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - labels_input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - target shape ids for computing the contrastive loss and masked_lm_loss . Indices should be in `[-100, - 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - labels_input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - target pronunciation ids for computing the contrastive loss and masked_lm_loss . Indices should be in - `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are - ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., - config.vocab_size]` + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. - kwargs (`Dict[str, any]`, *optional*, defaults to *{}*): - Used to hide legacy arguments that have been deprecated. + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. - Returns: + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) + attack_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + attack sample ids for computing the contrastive loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + attack_input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + attack sample shape ids for computing the contrastive loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + attack_input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + attack sample pronunciation ids for computing the contrastive loss. Indices should be in `[-100, 0, + ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + labels_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + target ids for computing the contrastive loss and masked_lm_loss . Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + labels_input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + target shape ids for computing the contrastive loss and masked_lm_loss . Indices should be in `[-100, + 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + labels_input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + target pronunciation ids for computing the contrastive loss and masked_lm_loss . Indices should be in + `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are + ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., + config.vocab_size]` + attack_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices for the attack sample. Mask values selected in + `[0, 1]`: `1` for tokens that are NOT MASKED, `0` for MASKED tokens. + attack_token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate different portions of the attack inputs. Indices are selected in `[0, 1]`: + `0` corresponds to a sentence A token, `1` corresponds to a sentence B token. + labels_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices for the label sample. Mask values selected in + `[0, 1]`: `1` for tokens that are NOT MASKED, `0` for MASKED tokens. + labels_token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate different portions of the label inputs. Indices are selected in `[0, 1]`: + `0` corresponds to a sentence A token, `1` corresponds to a sentence B token. Example: @@ -1264,7 +1165,7 @@ class RoCBertForPreTraining(RoCBertPreTrainedModel): ) -@add_start_docstrings("""RoCBert Model with a `language modeling` head on top.""", ROC_BERT_START_DOCSTRING) +@auto_docstring class RoCBertForMaskedLM(RoCBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -1293,7 +1194,7 @@ class RoCBertForMaskedLM(RoCBertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1312,6 +1213,20 @@ class RoCBertForMaskedLM(RoCBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the @@ -1403,8 +1318,10 @@ class RoCBertForMaskedLM(RoCBertPreTrainedModel): } -@add_start_docstrings( - """RoCBert Model with a `language modeling` head on top for CLM fine-tuning.""", ROC_BERT_START_DOCSTRING +@auto_docstring( + custom_intro=""" + RoCBert Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class RoCBertForCausalLM(RoCBertPreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -1431,8 +1348,7 @@ class RoCBertForCausalLM(RoCBertPreTrainedModel, GenerationMixin): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1454,36 +1370,24 @@ class RoCBertForCausalLM(RoCBertPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are - only required when the model is used as a decoder in a Sequence to Sequence model. + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1599,10 +1503,11 @@ class RoCBertForCausalLM(RoCBertPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """RoCBert Model transformer with a sequence classification/regression head on top (a linear layer on top of - the pooled output) e.g. for GLUE tasks.""", - ROC_BERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + RoCBert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. + """ ) class RoCBertForSequenceClassification(RoCBertPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForSequenceClassification.__init__ with Bert->RoCBert,bert->roc_bert @@ -1621,14 +1526,7 @@ class RoCBertForSequenceClassification(RoCBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1645,6 +1543,20 @@ class RoCBertForSequenceClassification(RoCBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1705,11 +1617,7 @@ class RoCBertForSequenceClassification(RoCBertPreTrainedModel): ) -@add_start_docstrings( - """RoCBert Model with a multiple choice classification head on top (a linear layer on top of - the pooled output and a softmax) e.g. for RocStories/SWAG tasks.""", - ROC_BERT_START_DOCSTRING, -) +@auto_docstring class RoCBertForMultipleChoice(RoCBertPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForMultipleChoice.__init__ with Bert->RoCBert,bert->roc_bert def __init__(self, config): @@ -1725,14 +1633,7 @@ class RoCBertForMultipleChoice(RoCBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - ROC_BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1749,6 +1650,44 @@ class RoCBertForMultipleChoice(RoCBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + input_shape_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1810,11 +1749,7 @@ class RoCBertForMultipleChoice(RoCBertPreTrainedModel): ) -@add_start_docstrings( - """RoCBert Model with a token classification head on top (a linear layer on top of - the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.""", - ROC_BERT_START_DOCSTRING, -) +@auto_docstring class RoCBertForTokenClassification(RoCBertPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForTokenClassification.__init__ with Bert->RoCBert,bert->roc_bert def __init__(self, config): @@ -1831,14 +1766,7 @@ class RoCBertForTokenClassification(RoCBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_TOKEN_CLASSIFICATION, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_TOKEN_CLASS_EXPECTED_OUTPUT, - expected_loss=_TOKEN_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1855,6 +1783,20 @@ class RoCBertForTokenClassification(RoCBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1896,11 +1838,7 @@ class RoCBertForTokenClassification(RoCBertPreTrainedModel): ) -@add_start_docstrings( - """RoCBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`).""", - ROC_BERT_START_DOCSTRING, -) +@auto_docstring class RoCBertForQuestionAnswering(RoCBertPreTrainedModel): # Copied from transformers.models.bert.modeling_bert.BertForQuestionAnswering.__init__ with Bert->RoCBert,bert->roc_bert def __init__(self, config): @@ -1913,16 +1851,7 @@ class RoCBertForQuestionAnswering(RoCBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROC_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_QA, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - qa_target_start_index=_QA_TARGET_START_INDEX, - qa_target_end_index=_QA_TARGET_END_INDEX, - expected_output=_QA_EXPECTED_OUTPUT, - expected_loss=_QA_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1940,14 +1869,20 @@ class RoCBertForQuestionAnswering(RoCBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + input_shape_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the shape vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_shape_ids) + input_pronunciation_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the pronunciation vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input_pronunciation_ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/roformer/modeling_roformer.py b/src/transformers/models/roformer/modeling_roformer.py index 86465153b1..7e4f1ff9f2 100644 --- a/src/transformers/models/roformer/modeling_roformer.py +++ b/src/transformers/models/roformer/modeling_roformer.py @@ -37,21 +37,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_roformer import RoFormerConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "junnyu/roformer_chinese_base" -_CONFIG_FOR_DOC = "RoFormerConfig" - # Copied from transformers.models.marian.modeling_marian.MarianSinusoidalPositionalEmbedding with Marian->RoFormer class RoFormerSinusoidalPositionalEmbedding(nn.Embedding): @@ -771,12 +762,8 @@ class RoFormerOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class RoFormerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RoFormerConfig load_tf_weights = load_tf_weights_in_roformer base_model_prefix = "roformer" @@ -803,68 +790,8 @@ class RoFormerPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -ROFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`RoFormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ROFORMER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RoFormer Model transformer outputting raw hidden-states without any specific head on top.", - ROFORMER_START_DOCSTRING, -) -class RoFormerModel(RoFormerPreTrainedModel): - """ +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is @@ -875,7 +802,8 @@ class RoFormerModel(RoFormerPreTrainedModel): to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. """ - +) +class RoFormerModel(RoFormerPreTrainedModel): def __init__(self, config): super().__init__(config) self.config = config @@ -903,12 +831,7 @@ class RoFormerModel(RoFormerPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -924,25 +847,6 @@ class RoFormerModel(RoFormerPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[BaseModelOutputWithPastAndCrossAttentions, Tuple[torch.Tensor]]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1029,7 +933,7 @@ class RoFormerModel(RoFormerPreTrainedModel): ) -@add_start_docstrings("""RoFormer Model with a `language modeling` head on top.""", ROFORMER_START_DOCSTRING) +@auto_docstring class RoFormerForMaskedLM(RoFormerPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1055,12 +959,7 @@ class RoFormerForMaskedLM(RoFormerPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1130,8 +1029,10 @@ class RoFormerForMaskedLM(RoFormerPreTrainedModel): return {"input_ids": input_ids, "attention_mask": attention_mask} -@add_start_docstrings( - """RoFormer Model with a `language modeling` head on top for CLM fine-tuning.""", ROFORMER_START_DOCSTRING +@auto_docstring( + custom_intro=""" + RoFormer Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class RoFormerForCausalLM(RoFormerPreTrainedModel, GenerationMixin): _tied_weights_keys = ["cls.predictions.decoder.bias", "cls.predictions.decoder.weight"] @@ -1155,8 +1056,7 @@ class RoFormerForCausalLM(RoFormerPreTrainedModel, GenerationMixin): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1176,29 +1076,10 @@ class RoFormerForCausalLM(RoFormerPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[CausalLMOutputWithCrossAttentions, Tuple[torch.Tensor]]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1289,12 +1170,11 @@ class RoFormerClassificationHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RoFormer Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - ROFORMER_START_DOCSTRING, + """ ) class RoFormerForSequenceClassification(RoFormerPreTrainedModel): def __init__(self, config): @@ -1306,12 +1186,7 @@ class RoFormerForSequenceClassification(RoFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1380,13 +1255,7 @@ class RoFormerForSequenceClassification(RoFormerPreTrainedModel): ) -@add_start_docstrings( - """ - RoFormer Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - ROFORMER_START_DOCSTRING, -) +@auto_docstring class RoFormerForMultipleChoice(RoFormerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1398,14 +1267,7 @@ class RoFormerForMultipleChoice(RoFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - ROFORMER_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1419,6 +1281,25 @@ class RoFormerForMultipleChoice(RoFormerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[MultipleChoiceModelOutput, Tuple[torch.Tensor]]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1471,13 +1352,7 @@ class RoFormerForMultipleChoice(RoFormerPreTrainedModel): ) -@add_start_docstrings( - """ - RoFormer Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - ROFORMER_START_DOCSTRING, -) +@auto_docstring class RoFormerForTokenClassification(RoFormerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1490,12 +1365,7 @@ class RoFormerForTokenClassification(RoFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1547,13 +1417,7 @@ class RoFormerForTokenClassification(RoFormerPreTrainedModel): ) -@add_start_docstrings( - """ - RoFormer Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - ROFORMER_START_DOCSTRING, -) +@auto_docstring class RoFormerForQuestionAnswering(RoFormerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1567,12 +1431,7 @@ class RoFormerForQuestionAnswering(RoFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ROFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1586,16 +1445,6 @@ class RoFormerForQuestionAnswering(RoFormerPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[QuestionAnsweringModelOutput, Tuple[torch.Tensor]]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.roformer( diff --git a/src/transformers/models/rt_detr/image_processing_rt_detr_fast.py b/src/transformers/models/rt_detr/image_processing_rt_detr_fast.py index e9bd83f1b2..75403e133d 100644 --- a/src/transformers/models/rt_detr/image_processing_rt_detr_fast.py +++ b/src/transformers/models/rt_detr/image_processing_rt_detr_fast.py @@ -9,12 +9,9 @@ from typing import Any, Dict, List, Optional, Tuple, Union from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, - add_start_docstrings, get_image_size_for_max_height_width, get_max_height_width, safe_squeeze, @@ -34,6 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -54,6 +52,26 @@ elif is_torchvision_available(): class RTDetrFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): + Data format of the annotations. One of "coco_detection" or "coco_panoptic". + do_convert_annotations (`bool`, *optional*, defaults to `True`): + Controls whether to convert the annotations to the format expected by the RT_DETR model. Converts the + bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. + Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with zeros. + If `pad_size` is provided, the image will be padded to the specified dimensions. + Otherwise, the image will be padded to the maximum height and width of the batch. + pad_size (`Dict[str, int]`, *optional*): + The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size + provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest + height and width in the batch. + return_segmentation_masks (`bool`, *optional*, defaults to `False`): + Whether to return segmentation masks. + """ + format: Optional[Union[str, AnnotationFormat]] do_convert_annotations: Optional[bool] do_pad: Optional[bool] @@ -123,29 +141,7 @@ def prepare_coco_detection_annotation( return new_target -@add_start_docstrings( - "Constructs a fast RTDetr image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the RT_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - """, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class RTDetrImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR @@ -381,9 +377,15 @@ class RTDetrImageProcessorFast(BaseImageProcessorFast): return image, pixel_mask, annotation - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ + @auto_docstring + def preprocess( + self, + images: ImageInput, + annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, + masks_path: Optional[Union[str, pathlib.Path]] = None, + **kwargs: Unpack[RTDetrFastImageProcessorKwargs], + ) -> BatchFeature: + r""" annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: @@ -397,32 +399,9 @@ class RTDetrImageProcessorFast(BaseImageProcessorFast): - "file_name" (`str`): The file name of the image. format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. masks_path (`str` or `pathlib.Path`, *optional*): Path to the directory containing the segmentation masks. - """, - ) - def preprocess( - self, - images: ImageInput, - annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, - masks_path: Optional[Union[str, pathlib.Path]] = None, - **kwargs: Unpack[RTDetrFastImageProcessorKwargs], - ) -> BatchFeature: + """ return super().preprocess(images, annotations=annotations, masks_path=masks_path, **kwargs) def _preprocess( diff --git a/src/transformers/models/rt_detr/modeling_rt_detr.py b/src/transformers/models/rt_detr/modeling_rt_detr.py index d3dda3c12b..59bcf08578 100644 --- a/src/transformers/models/rt_detr/modeling_rt_detr.py +++ b/src/transformers/models/rt_detr/modeling_rt_detr.py @@ -32,10 +32,8 @@ from ...modeling_utils import PreTrainedModel from ...pytorch_utils import compile_compatible_method_lru_cache from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, torch_int, ) from ...utils.backbone_utils import load_backbone @@ -45,9 +43,7 @@ from .configuration_rt_detr import RTDetrConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "RTDetrConfig" # TODO: Replace all occurrences of the checkpoint with the final one -_CHECKPOINT_FOR_DOC = "PekingU/rtdetr_r50vd" @use_kernel_forward_from_hub("MultiScaleDeformableAttention") @@ -1046,6 +1042,7 @@ class RTDetrDecoderLayer(nn.Module): return outputs +@auto_docstring class RTDetrPreTrainedModel(PreTrainedModel): config_class = RTDetrConfig base_model_prefix = "rt_detr" @@ -1111,60 +1108,6 @@ class RTDetrPreTrainedModel(PreTrainedModel): nn.init.xavier_uniform_(module.denoising_class_embed.weight) -RTDETR_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`RTDetrConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -RTDETR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`RTDetrImageProcessor.__call__`] for details. - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - labels (`List[Dict]` of len `(batch_size,)`, *optional*): - Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the - following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch - respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes - in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class RTDetrEncoder(nn.Module): def __init__(self, config: RTDetrConfig): super().__init__() @@ -1559,11 +1502,10 @@ class RTDetrMLPPredictionHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RT-DETR Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. - """, - RTDETR_START_DOCSTRING, + """ ) class RTDetrModel(RTDetrPreTrainedModel): def __init__(self, config: RTDetrConfig): @@ -1682,8 +1624,7 @@ class RTDetrModel(RTDetrPreTrainedModel): return anchors, valid_mask - @add_start_docstrings_to_model_forward(RTDETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RTDetrModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1697,7 +1638,17 @@ class RTDetrModel(RTDetrPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], RTDetrModelOutput]: r""" - Returns: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`List[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. Examples: @@ -1896,12 +1847,11 @@ class RTDetrModel(RTDetrPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RT-DETR Model (consisting of a backbone and encoder-decoder) outputting bounding boxes and logits to be further decoded into scores and classes. - """, - RTDETR_START_DOCSTRING, + """ ) class RTDetrForObjectDetection(RTDetrPreTrainedModel): # When using clones, all layers > 0 will be clones, but layer 0 *is* required @@ -1942,8 +1892,7 @@ class RTDetrForObjectDetection(RTDetrPreTrainedModel): # as a dict having both a Tensor and a list. return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class, outputs_coord)] - @add_start_docstrings_to_model_forward(RTDETR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RTDetrObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1958,14 +1907,18 @@ class RTDetrForObjectDetection(RTDetrPreTrainedModel): **loss_kwargs, ) -> Union[Tuple[torch.FloatTensor], RTDetrObjectDetectionOutput]: r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/rt_detr/modeling_rt_detr_resnet.py b/src/transformers/models/rt_detr/modeling_rt_detr_resnet.py index 72ef9a0ac4..17833c7037 100644 --- a/src/transformers/models/rt_detr/modeling_rt_detr_resnet.py +++ b/src/transformers/models/rt_detr/modeling_rt_detr_resnet.py @@ -23,30 +23,15 @@ from typing import Optional from torch import Tensor, nn from ...activations import ACT2FN -from ...modeling_outputs import ( - BackboneOutput, - BaseModelOutputWithNoAttention, -) +from ...modeling_outputs import BackboneOutput, BaseModelOutputWithNoAttention from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_rt_detr_resnet import RTDetrResNetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "RTDetrResNetConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/resnet-50" -_EXPECTED_OUTPUT_SHAPE = [1, 2048, 7, 7] - # Copied from transformers.models.resnet.modeling_resnet.ResNetConvLayer -> RTDetrResNetConvLayer class RTDetrResNetConvLayer(nn.Module): @@ -309,13 +294,9 @@ class RTDetrResNetEncoder(nn.Module): ) +@auto_docstring # Copied from transformers.models.resnet.modeling_resnet.ResNetPreTrainedModel with ResNet->RTDetrResNet class RTDetrResNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RTDetrResNetConfig base_model_prefix = "resnet" main_input_name = "pixel_values" @@ -336,36 +317,10 @@ class RTDetrResNetPreTrainedModel(PreTrainedModel): nn.init.constant_(module.bias, 0) -RTDETR_RESNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`RTDetrResNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -RTDETR_RESNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`RTDetrImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ResNet backbone, to be used with frameworks like RTDETR. - """, - RTDETR_RESNET_START_DOCSTRING, + """ ) class RTDetrResNetBackbone(RTDetrResNetPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -379,14 +334,11 @@ class RTDetrResNetBackbone(RTDetrResNetPreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(RTDETR_RESNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/rt_detr/modular_rt_detr.py b/src/transformers/models/rt_detr/modular_rt_detr.py index fd9913e97e..15408b5edc 100644 --- a/src/transformers/models/rt_detr/modular_rt_detr.py +++ b/src/transformers/models/rt_detr/modular_rt_detr.py @@ -1,19 +1,10 @@ import pathlib from typing import Dict, List, Optional, Tuple, Union -from transformers.models.detr.image_processing_detr_fast import ( - DetrFastImageProcessorKwargs, - DetrImageProcessorFast, -) +from transformers.models.detr.image_processing_detr_fast import DetrFastImageProcessorKwargs, DetrImageProcessorFast from ...image_processing_utils import BatchFeature -from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - BaseImageProcessorFast, - SizeDict, - add_start_docstrings, - get_max_height_width, -) +from ...image_processing_utils_fast import BaseImageProcessorFast, SizeDict, get_max_height_width from ...image_transforms import center_to_corners_format from ...image_utils import ( IMAGENET_DEFAULT_MEAN, @@ -139,41 +130,6 @@ class RTDetrImageProcessorFast(DetrImageProcessorFast, BaseImageProcessorFast): BaseImageProcessorFast.__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): - List of annotations associated with the image or batch of images. If annotation is for object - detection, the annotations should be a dictionary with the following keys: - - "image_id" (`int`): The image id. - - "annotations" (`List[Dict]`): List of annotations for an image. Each annotation should be a - dictionary. An image can have no annotations, in which case the list should be empty. - If annotation is for segmentation, the annotations should be a dictionary with the following keys: - - "image_id" (`int`): The image id. - - "segments_info" (`List[Dict]`): List of segments for an image. Each segment should be a dictionary. - An image can have no segments, in which case the list should be empty. - - "file_name" (`str`): The file name of the image. - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - masks_path (`str` or `pathlib.Path`, *optional*): - Path to the directory containing the segmentation masks. - """, - ) def preprocess( self, images: ImageInput, diff --git a/src/transformers/models/rt_detr_v2/modeling_rt_detr_v2.py b/src/transformers/models/rt_detr_v2/modeling_rt_detr_v2.py index ab47e5be68..d8a145d645 100644 --- a/src/transformers/models/rt_detr_v2/modeling_rt_detr_v2.py +++ b/src/transformers/models/rt_detr_v2/modeling_rt_detr_v2.py @@ -33,21 +33,11 @@ from ...image_transforms import center_to_corners_format, corners_to_center_form from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import compile_compatible_method_lru_cache -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torchdynamo_compiling, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, is_torchdynamo_compiling, torch_int from ...utils.backbone_utils import load_backbone from .configuration_rt_detr_v2 import RTDetrV2Config -_CONFIG_FOR_DOC = "RTDetrV2Config" - - def multi_scale_deformable_attention_v2( value: Tensor, value_spatial_shapes: Tensor, @@ -1262,64 +1252,11 @@ def get_contrastive_denoising_training_group( return input_query_class, input_query_bbox, attn_mask, denoising_meta_values -RTDetrV2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`RTDetrV2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -RTDetrV2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`RTDetrV2ImageProcessor.__call__`] for details. - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - labels (`List[Dict]` of len `(batch_size,)`, *optional*): - Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the - following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch - respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes - in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - def _get_clones(partial_module, N): return nn.ModuleList([partial_module() for i in range(N)]) +@auto_docstring class RTDetrV2PreTrainedModel(PreTrainedModel): config_class = RTDetrV2Config base_model_prefix = "rt_detr_v2" @@ -1544,11 +1481,10 @@ class RTDetrV2Decoder(RTDetrV2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RT-DETR Model (consisting of a backbone and encoder-decoder) outputting raw hidden states without any head on top. - """, - RTDetrV2_START_DOCSTRING, + """ ) class RTDetrV2Model(RTDetrV2PreTrainedModel): def __init__(self, config: RTDetrV2Config): @@ -1666,8 +1602,7 @@ class RTDetrV2Model(RTDetrV2PreTrainedModel): return anchors, valid_mask - @add_start_docstrings_to_model_forward(RTDetrV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RTDetrV2ModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1681,7 +1616,17 @@ class RTDetrV2Model(RTDetrV2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], RTDetrV2ModelOutput]: r""" - Returns: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. + labels (`List[Dict]` of len `(batch_size,)`, *optional*): + Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the + following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch + respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes + in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. Examples: @@ -1903,12 +1848,11 @@ class RTDetrV2MLPPredictionHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" RT-DETR Model (consisting of a backbone and encoder-decoder) outputting bounding boxes and logits to be further decoded into scores and classes. - """, - RTDetrV2_START_DOCSTRING, + """ ) class RTDetrV2ForObjectDetection(RTDetrV2PreTrainedModel): # When using clones, all layers > 0 will be clones, but layer 0 *is* required @@ -1941,8 +1885,7 @@ class RTDetrV2ForObjectDetection(RTDetrV2PreTrainedModel): # as a dict having both a Tensor and a list. return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class, outputs_coord)] - @add_start_docstrings_to_model_forward(RTDetrV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=RTDetrV2ObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1957,14 +1900,18 @@ class RTDetrV2ForObjectDetection(RTDetrV2PreTrainedModel): **loss_kwargs, ) -> Union[Tuple[torch.FloatTensor], RTDetrV2ObjectDetectionOutput]: r""" + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/rwkv/modeling_rwkv.py b/src/transformers/models/rwkv/modeling_rwkv.py index 6274464309..6e71308e17 100644 --- a/src/transformers/models/rwkv/modeling_rwkv.py +++ b/src/transformers/models/rwkv/modeling_rwkv.py @@ -28,9 +28,7 @@ from ...generation import GenerationMixin from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_bitsandbytes_available, is_ninja_available, is_torch_cuda_available, @@ -41,9 +39,6 @@ from .configuration_rwkv import RwkvConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "RWKV/rwkv-4-169m-pile" -_CONFIG_FOR_DOC = "RwkvConfig" - rwkv_cuda_kernel = None @@ -383,12 +378,8 @@ class RwkvBlock(nn.Module): return outputs +@auto_docstring class RwkvPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = RwkvConfig base_model_prefix = "rwkv" _no_split_modules = ["RwkvBlock"] @@ -517,69 +508,7 @@ class RwkvCausalLMOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor, ...]] = None -RWKV_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`RwkvConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -RWKV_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): - `input_ids_length` = `sequence_length` if `past_key_values` is `None` else - `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input - sequence tokens in the vocabulary. - - If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as - `input_ids`. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - This is currently not used by `RwkvModel`, but will be supported in the future. - - [What are attention masks?](../glossary#attention-mask) - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): - If passed along, the model uses the previous state in all the blocks (which will give the output for the - `input_ids` provided as if the model add `state_input_ids + input_ids` as context). - use_cache (`bool`, *optional*): - If set to `True`, the last state is returned and can be used to quickly generate the next logits. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare RWKV Model transformer outputting raw hidden-states without any specific head on top.", - RWKV_START_DOCSTRING, -) +@auto_docstring class RwkvModel(RwkvPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -601,12 +530,7 @@ class RwkvModel(RwkvPreTrainedModel): def set_input_embeddings(self, new_embeddings): self.embeddings = new_embeddings - @add_start_docstrings_to_model_forward(RWKV_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=RwkvOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -618,6 +542,25 @@ class RwkvModel(RwkvPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, RwkvOutput]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). + use_cache (`bool`, *optional*): + If set to `True`, the last state is returned and can be used to quickly generate the next logits. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -744,12 +687,11 @@ class RwkvModel(RwkvPreTrainedModel): setattr(target_layer, "weight", quant_weight) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - RWKV_START_DOCSTRING, + """ ) class RwkvForCausalLM(RwkvPreTrainedModel, GenerationMixin): _tied_weights_keys = ["head.weight"] @@ -785,12 +727,7 @@ class RwkvForCausalLM(RwkvPreTrainedModel, GenerationMixin): model_inputs["use_cache"] = use_cache return model_inputs - @add_start_docstrings_to_model_forward(RWKV_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=RwkvCausalLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -805,10 +742,27 @@ class RwkvForCausalLM(RwkvPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, RwkvCausalLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): + `input_ids_length` = `sequence_length` if `past_key_values` is `None` else + `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input + sequence tokens in the vocabulary. + + If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as + `input_ids`. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): + If passed along, the model uses the previous state in all the blocks (which will give the output for the + `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + use_cache (`bool`, *optional*): + If set to `True`, the last state is returned and can be used to quickly generate the next logits. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/sam/modeling_sam.py b/src/transformers/models/sam/modeling_sam.py index c81b351c8c..f49280e717 100644 --- a/src/transformers/models/sam/modeling_sam.py +++ b/src/transformers/models/sam/modeling_sam.py @@ -29,20 +29,15 @@ from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, can_return_tuple, logging, - replace_return_docstrings, ) from .configuration_sam import SamConfig, SamMaskDecoderConfig, SamPromptEncoderConfig, SamVisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "SamConfig" -_CHECKPOINT_FOR_DOC = "facebook/sam-vit-huge" - @dataclass class SamVisionEncoderOutput(ModelOutput): @@ -1175,6 +1170,7 @@ class SamVisionEncoder(nn.Module): ) +@auto_docstring class SamPreTrainedModel(PreTrainedModel): config_class = SamConfig base_model_prefix = "sam" @@ -1202,111 +1198,10 @@ class SamPreTrainedModel(PreTrainedModel): module.rel_pos_w.data.zero_() -SAM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SamConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -SAM_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SamProcessor`]. See [`SamProcessor.__call__`] for - details. - input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): - Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much - better results. The points can be obtained by passing a list of list of list to the processor that will - create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the - second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict - per input point), the third dimension is the number of points per segmentation mask (it is possible to pass - multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) - coordinates of the point. If a different number of points is passed either for each image, or for each - mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the - computation of the embedding will be skipped for these points using the labels. - input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): - Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the - official implementation, there are 3 types of labels - - - `1`: the point is a point that contains the object of interest - - `0`: the point is a point that does not contain the object of interest - - `-1`: the point corresponds to the background - - We added the label: - - - `-10`: the point is a padding point, thus should be ignored by the prompt encoder - - The padding labels should be automatically done by the processor. - input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): - Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to - much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, - that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch - size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. - In the order (`x1`, `y1`, `x2`, `y2`): - - - `x1`: the x coordinate of the top left point of the input box - - `y1`: the y coordinate of the top left point of the input box - - `x2`: the x coordinate of the bottom right point of the input box - - `y2`: the y coordinate of the bottom right point of the input box - - input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): - SAM model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to - generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be - manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). - - image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): - Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory - efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` - method, and then feed them to the `forward` method instead of feeding the `pixel_values`. - multimask_output (`bool`, *optional*): - In the original implementation and paper, the model always outputs 3 masks per image (or per point / per - bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the - "best" mask, by specifying `multimask_output=False`. - attention_similarity (`torch.FloatTensor`, *optional*): - Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the - model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - target_embedding (`torch.FloatTensor`, *optional*): - Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case - the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -SAM_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SamProcessor`]. See [`SamProcessor.__call__`] for - details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """The vision model from Sam without any head or projection on top.""", - SAM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from Sam without any head or projection on top. + """ ) class SamVisionModel(SamPreTrainedModel): config_class = SamVisionConfig @@ -1322,8 +1217,7 @@ class SamVisionModel(SamPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_encoder.patch_embed - @add_start_docstrings_to_model_forward(SAM_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SamVisionEncoderOutput, config_class=SamVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1331,10 +1225,6 @@ class SamVisionModel(SamPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, SamVisionEncoderOutput]: - r""" - Returns: - - """ return self.vision_encoder( pixel_values, output_attentions=output_attentions, @@ -1343,10 +1233,10 @@ class SamVisionModel(SamPreTrainedModel): ) -@add_start_docstrings( - "Segment Anything Model (SAM) for generating segmentation masks, given an input image and ", - " optional 2D location and bounding boxes.", - SAM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Segment Anything Model (SAM) for generating segmentation masks, given an input image and + """ ) class SamModel(SamPreTrainedModel): _tied_weights_keys = ["prompt_encoder.shared_embedding.positional_embedding"] @@ -1444,7 +1334,7 @@ class SamModel(SamPreTrainedModel): return prompt_output @can_return_tuple - @add_start_docstrings_to_model_forward(SAM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1461,6 +1351,59 @@ class SamModel(SamPreTrainedModel): **kwargs, ) -> SamImageSegmentationOutput: r""" + input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): + Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much + better results. The points can be obtained by passing a list of list of list to the processor that will + create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the + second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict + per input point), the third dimension is the number of points per segmentation mask (it is possible to pass + multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) + coordinates of the point. If a different number of points is passed either for each image, or for each + mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the + computation of the embedding will be skipped for these points using the labels. + input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): + Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the + official implementation, there are 3 types of labels + + - `1`: the point is a point that contains the object of interest + - `0`: the point is a point that does not contain the object of interest + - `-1`: the point corresponds to the background + + We added the label: + + - `-10`: the point is a padding point, thus should be ignored by the prompt encoder + + The padding labels should be automatically done by the processor. + input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): + Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to + much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, + that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch + size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. + In the order (`x1`, `y1`, `x2`, `y2`): + + - `x1`: the x coordinate of the top left point of the input box + - `y1`: the y coordinate of the top left point of the input box + - `x2`: the x coordinate of the bottom right point of the input box + - `y2`: the y coordinate of the bottom right point of the input box + input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): + SAM model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to + generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be + manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). + image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): + Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory + efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` + method, and then feed them to the `forward` method instead of feeding the `pixel_values`. + multimask_output (`bool`, *optional*): + In the original implementation and paper, the model always outputs 3 masks per image (or per point / per + bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the + "best" mask, by specifying `multimask_output=False`. + attention_similarity (`torch.FloatTensor`, *optional*): + Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the + model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). + target_embedding (`torch.FloatTensor`, *optional*): + Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case + the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). + Example: ```python diff --git a/src/transformers/models/sam_hq/modeling_sam_hq.py b/src/transformers/models/sam_hq/modeling_sam_hq.py index 80a276b5bd..0a77f996cd 100644 --- a/src/transformers/models/sam_hq/modeling_sam_hq.py +++ b/src/transformers/models/sam_hq/modeling_sam_hq.py @@ -31,14 +31,7 @@ from torch import Tensor, nn from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging from .configuration_sam_hq import SamHQConfig, SamHQMaskDecoderConfig, SamHQPromptEncoderConfig, SamHQVisionConfig @@ -1121,6 +1114,7 @@ class SamHQMaskDecoder(nn.Module): return outputs +@auto_docstring class SamHQPreTrainedModel(PreTrainedModel): config_class = SamHQConfig base_model_prefix = "sam_hq" @@ -1151,41 +1145,10 @@ class SamHQPreTrainedModel(PreTrainedModel): module.pos_embed.data.zero_() -SAM_HQ_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SamHQProcessor`]. See [`SamHQProcessor.__call__`] for - details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -SAM_HQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SamHQConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The vision model from SAM-HQ without any head or projection on top.""", - SAM_HQ_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from SamHQ without any head or projection on top. + """ ) class SamHQVisionModel(SamHQPreTrainedModel): config_class = SamHQVisionConfig @@ -1201,8 +1164,7 @@ class SamHQVisionModel(SamHQPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_encoder.patch_embed - @add_start_docstrings_to_model_forward(SAM_HQ_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SamHQVisionEncoderOutput, config_class=SamHQVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1210,10 +1172,6 @@ class SamHQVisionModel(SamHQPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, SamHQVisionEncoderOutput]: - r""" - Returns: - - """ return self.vision_encoder( pixel_values, output_attentions=output_attentions, @@ -1385,80 +1343,10 @@ class SamHQPromptEncoder(nn.Module): return sparse_embeddings, dense_embeddings -SAM_HQ_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SamHQProcessor`]. See [`SamHQProcessor.__call__`] for - details. - input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): - Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much - better results. The points can be obtained by passing a list of list of list to the processor that will - create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the - second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict - per input point), the third dimension is the number of points per segmentation mask (it is possible to pass - multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) - coordinates of the point. If a different number of points is passed either for each image, or for each - mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the - computation of the embedding will be skipped for these points using the labels. - input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): - Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the - official implementation, there are 3 types of labels - - - `1`: the point is a point that contains the object of interest - - `0`: the point is a point that does not contain the object of interest - - `-1`: the point corresponds to the background - - We added the label: - - - `-10`: the point is a padding point, thus should be ignored by the prompt encoder - - The padding labels should be automatically done by the processor. - input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): - Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to - much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, - that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch - size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. - In the order (`x1`, `y1`, `x2`, `y2`): - - - `x1`: the x coordinate of the top left point of the input box - - `y1`: the y coordinate of the top left point of the input box - - `x2`: the x coordinate of the bottom right point of the input box - - `y2`: the y coordinate of the bottom right point of the input box - - input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): - SAM_HQ model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to - generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be - manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). - - image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): - Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory - efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` - method, and then feed them to the `forward` method instead of feeding the `pixel_values`. - multimask_output (`bool`, *optional*): - In the original implementation and paper, the model always outputs 3 masks per image (or per point / per - bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the - "best" mask, by specifying `multimask_output=False`. - attention_similarity (`torch.FloatTensor`, *optional*): - Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the - model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - target_embedding (`torch.FloatTensor`, *optional*): - Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case - the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "Segment Anything Model HQ (SAM-HQ) for generating masks,given an input image and", - " optional 2D location and bounding boxes.", - SAM_HQ_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Segment Anything Model HQ (SAM-HQ) for generating masks, given an input image and optional 2D location and bounding boxes. + """ ) class SamHQModel(SamHQPreTrainedModel): _tied_weights_keys = ["prompt_encoder.shared_embedding.positional_embedding"] @@ -1563,7 +1451,7 @@ class SamHQModel(SamHQPreTrainedModel): return prompt_output @can_return_tuple - @add_start_docstrings_to_model_forward(SAM_HQ_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1583,80 +1471,67 @@ class SamHQModel(SamHQPreTrainedModel): **kwargs, ) -> List[Dict[str, torch.Tensor]]: r""" + input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): + Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much + better results. The points can be obtained by passing a list of list of list to the processor that will + create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the + second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict + per input point), the third dimension is the number of points per segmentation mask (it is possible to pass + multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) + coordinates of the point. If a different number of points is passed either for each image, or for each + mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the + computation of the embedding will be skipped for these points using the labels. + input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): + Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the + official implementation, there are 3 types of labels - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SamHQProcessor`]. See [`SamHQProcessor.__call__`] for - details. - input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): - Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much - better results. The points can be obtained by passing a list of list of list to the processor that will - create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the - second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict - per input point), the third dimension is the number of points per segmentation mask (it is possible to pass - multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) - coordinates of the point. If a different number of points is passed either for each image, or for each - mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the - computation of the embedding will be skipped for these points using the labels. - input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): - Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the - official implementation, there are 3 types of labels + - `1`: the point is a point that contains the object of interest + - `0`: the point is a point that does not contain the object of interest + - `-1`: the point corresponds to the background - - `1`: the point is a point that contains the object of interest - - `0`: the point is a point that does not contain the object of interest - - `-1`: the point corresponds to the background + We added the label: - We added the label: + - `-10`: the point is a padding point, thus should be ignored by the prompt encoder - - `-10`: the point is a padding point, thus should be ignored by the prompt encoder + The padding labels should be automatically done by the processor. + input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): + Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to + much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, + that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch + size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. + In the order (`x1`, `y1`, `x2`, `y2`): - The padding labels should be automatically done by the processor. - input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): - Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to - much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, - that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch - size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. - In the order (`x1`, `y1`, `x2`, `y2`): + - `x1`: the x coordinate of the top left point of the input box + - `y1`: the y coordinate of the top left point of the input box + - `x2`: the x coordinate of the bottom right point of the input box + - `y2`: the y coordinate of the bottom right point of the input box + input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): + SAM_HQ model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to + generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be + manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). + image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): + Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory + efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` + method, and then feed them to the `forward` method instead of feeding the `pixel_values`. + multimask_output (`bool`, *optional*): + In the original implementation and paper, the model always outputs 3 masks per image (or per point / per + bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the + "best" mask, by specifying `multimask_output=False`. + hq_token_only (`bool`, *optional*, defaults to `False`): + Whether to use only the HQ token path for mask generation. When False, combines both standard and HQ paths. + This is specific to SAM-HQ's architecture. + attention_similarity (`torch.FloatTensor`, *optional*): + Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the + model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). + target_embedding (`torch.FloatTensor`, *optional*): + Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case + the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). + intermediate_embeddings (`List[torch.FloatTensor]`, *optional*): + Intermediate embeddings from vision encoder's non-windowed blocks, used by SAM-HQ for enhanced mask quality. + Required when providing pre-computed image_embeddings instead of pixel_values. - - `x1`: the x coordinate of the top left point of the input box - - `y1`: the y coordinate of the top left point of the input box - - `x2`: the x coordinate of the bottom right point of the input box - - `y2`: the y coordinate of the bottom right point of the input box - - input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): - SAM_HQ model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to - generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be - manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). - - image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): - Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory - efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` - method, and then feed them to the `forward` method instead of feeding the `pixel_values`. - multimask_output (`bool`, *optional*): - In the original implementation and paper, the model always outputs 3 masks per image (or per point / per - bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the - "best" mask, by specifying `multimask_output=False`. - hq_token_only (`bool`, *optional*, defaults to `False`): - Whether to use only the HQ token path for mask generation. When False, combines both standard and HQ paths. - This is specific to SAM-HQ's architecture. - attention_similarity (`torch.FloatTensor`, *optional*): - Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the - model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - target_embedding (`torch.FloatTensor`, *optional*): - Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case - the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - intermediate_embeddings (`List[torch.FloatTensor]`, *optional*): - Intermediate embeddings from vision encoder's non-windowed blocks, used by SAM-HQ for enhanced mask quality. - Required when providing pre-computed image_embeddings instead of pixel_values. Example: + ```python >>> from PIL import Image >>> import requests diff --git a/src/transformers/models/sam_hq/modular_sam_hq.py b/src/transformers/models/sam_hq/modular_sam_hq.py index 50b5f2aecc..e57eaffbb9 100644 --- a/src/transformers/models/sam_hq/modular_sam_hq.py +++ b/src/transformers/models/sam_hq/modular_sam_hq.py @@ -20,7 +20,7 @@ import torch import torch.utils.checkpoint from torch import nn -from ...utils import add_start_docstrings, logging +from ...utils import auto_docstring, logging from ..sam.configuration_sam import SamConfig, SamMaskDecoderConfig, SamPromptEncoderConfig, SamVisionConfig from ..sam.modeling_sam import ( SamFeedForward, @@ -425,34 +425,14 @@ class SamHQPreTrainedModel(SamPreTrainedModel): module.pos_embed.data.zero_() -SAM_HQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SamHQConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The vision model from SAM-HQ without any head or projection on top.""", - SAM_HQ_START_DOCSTRING, -) class SamHQVisionModel(SamVisionModel): pass -@add_start_docstrings( - "Segment Anything Model HQ (SAM-HQ) for generating masks,given an input image and", - " optional 2D location and bounding boxes.", - SAM_HQ_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Segment Anything Model HQ (SAM-HQ) for generating masks, given an input image and optional 2D location and bounding boxes. + """ ) class SamHQModel(SamModel): _tied_weights_keys = ["prompt_encoder.shared_embedding.positional_embedding"] @@ -519,80 +499,67 @@ class SamHQModel(SamModel): **kwargs, ) -> List[Dict[str, torch.Tensor]]: r""" + input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): + Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much + better results. The points can be obtained by passing a list of list of list to the processor that will + create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the + second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict + per input point), the third dimension is the number of points per segmentation mask (it is possible to pass + multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) + coordinates of the point. If a different number of points is passed either for each image, or for each + mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the + computation of the embedding will be skipped for these points using the labels. + input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): + Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the + official implementation, there are 3 types of labels - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SamHQProcessor`]. See [`SamHQProcessor.__call__`] for - details. - input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`): - Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much - better results. The points can be obtained by passing a list of list of list to the processor that will - create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the - second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict - per input point), the third dimension is the number of points per segmentation mask (it is possible to pass - multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal) - coordinates of the point. If a different number of points is passed either for each image, or for each - mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the - computation of the embedding will be skipped for these points using the labels. - input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`): - Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the - official implementation, there are 3 types of labels + - `1`: the point is a point that contains the object of interest + - `0`: the point is a point that does not contain the object of interest + - `-1`: the point corresponds to the background - - `1`: the point is a point that contains the object of interest - - `0`: the point is a point that does not contain the object of interest - - `-1`: the point corresponds to the background + We added the label: - We added the label: + - `-10`: the point is a padding point, thus should be ignored by the prompt encoder - - `-10`: the point is a padding point, thus should be ignored by the prompt encoder + The padding labels should be automatically done by the processor. + input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): + Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to + much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, + that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch + size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. + In the order (`x1`, `y1`, `x2`, `y2`): - The padding labels should be automatically done by the processor. - input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`): - Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to - much better generated masks. The boxes can be obtained by passing a list of list of list to the processor, - that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch - size, the number of boxes per image and the coordinates of the top left and bottom right point of the box. - In the order (`x1`, `y1`, `x2`, `y2`): + - `x1`: the x coordinate of the top left point of the input box + - `y1`: the y coordinate of the top left point of the input box + - `x2`: the x coordinate of the bottom right point of the input box + - `y2`: the y coordinate of the bottom right point of the input box + input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): + SAM_HQ model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to + generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be + manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). + image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): + Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory + efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` + method, and then feed them to the `forward` method instead of feeding the `pixel_values`. + multimask_output (`bool`, *optional*): + In the original implementation and paper, the model always outputs 3 masks per image (or per point / per + bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the + "best" mask, by specifying `multimask_output=False`. + hq_token_only (`bool`, *optional*, defaults to `False`): + Whether to use only the HQ token path for mask generation. When False, combines both standard and HQ paths. + This is specific to SAM-HQ's architecture. + attention_similarity (`torch.FloatTensor`, *optional*): + Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the + model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). + target_embedding (`torch.FloatTensor`, *optional*): + Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case + the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). + intermediate_embeddings (`List[torch.FloatTensor]`, *optional*): + Intermediate embeddings from vision encoder's non-windowed blocks, used by SAM-HQ for enhanced mask quality. + Required when providing pre-computed image_embeddings instead of pixel_values. - - `x1`: the x coordinate of the top left point of the input box - - `y1`: the y coordinate of the top left point of the input box - - `x2`: the x coordinate of the bottom right point of the input box - - `y2`: the y coordinate of the bottom right point of the input box - - input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`): - SAM_HQ model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to - generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be - manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`). - - image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`): - Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory - efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings` - method, and then feed them to the `forward` method instead of feeding the `pixel_values`. - multimask_output (`bool`, *optional*): - In the original implementation and paper, the model always outputs 3 masks per image (or per point / per - bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the - "best" mask, by specifying `multimask_output=False`. - hq_token_only (`bool`, *optional*, defaults to `False`): - Whether to use only the HQ token path for mask generation. When False, combines both standard and HQ paths. - This is specific to SAM-HQ's architecture. - attention_similarity (`torch.FloatTensor`, *optional*): - Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the - model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - target_embedding (`torch.FloatTensor`, *optional*): - Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case - the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - intermediate_embeddings (`List[torch.FloatTensor]`, *optional*): - Intermediate embeddings from vision encoder's non-windowed blocks, used by SAM-HQ for enhanced mask quality. - Required when providing pre-computed image_embeddings instead of pixel_values. Example: + ```python >>> from PIL import Image >>> import requests diff --git a/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py b/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py index d610bc6657..d2feaafa85 100755 --- a/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py +++ b/src/transformers/models/seamless_m4t/modeling_seamless_m4t.py @@ -37,19 +37,42 @@ from ...modeling_outputs import ( Wav2Vec2BaseModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_seamless_m4t import SeamlessM4TConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/hf-seamless-m4t-medium" -_CONFIG_FOR_DOC = "SeamlessM4TConfig" +SEAMLESS_M4T_COMMON_CUSTOM_ARGS = r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_banks)`): + Input audio features. This should be returned by the [`SeamlessM4TFeatureExtractor`] class or the + [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + inputs_embeds (`torch.FloatTensor` of shape`(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. +""" @dataclass @@ -79,129 +102,6 @@ class SeamlessM4TGenerationOutput(ModelOutput): unit_sequences: Optional[Tuple[torch.FloatTensor]] = None -SEAMLESS_M4T_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~SeamlessM4TConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SEAMLESS_M4T_INPUTS_DOCSTRING_FIRST_PART = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_banks)`): - Input audio features. This should be returned by the [`SeamlessM4TFeatureExtractor`] class or the - [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details. - """ - -SEAMLESS_M4T_INPUTS_DOCSTRING_TEXT_PART = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - """ - -SEAMLESS_M4T_INPUTS_DOCSTRING_SPEECH_PART = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_banks)`): - Input audio features. This should be returned by the [`SeamlessM4TFeatureExtractor`] class or the - [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details. - """ - -SEAMLESS_M4T_INPUTS_DOCSTRING_LAST_PART = r""" - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape`(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the - loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -M4T_MODEL_INPUTS_DOCSTRING = SEAMLESS_M4T_INPUTS_DOCSTRING_FIRST_PART + SEAMLESS_M4T_INPUTS_DOCSTRING_LAST_PART - -M4T_TEXT_INPUTS_DOCSTRING = SEAMLESS_M4T_INPUTS_DOCSTRING_TEXT_PART + SEAMLESS_M4T_INPUTS_DOCSTRING_LAST_PART - -M4T_SPEECH_INPUTS_DOCSTRING = SEAMLESS_M4T_INPUTS_DOCSTRING_SPEECH_PART + SEAMLESS_M4T_INPUTS_DOCSTRING_LAST_PART - - ############ UTILS ################ @@ -1443,12 +1343,8 @@ class SeamlessM4TDecoderLayer(nn.Module): ############ SUB-MODELS related code ################ +@auto_docstring class SeamlessM4TPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SeamlessM4TConfig base_model_prefix = "seamless_m4t" supports_gradient_checkpointing = True @@ -1553,10 +1449,11 @@ class SeamlessM4TPreTrainedModel(PreTrainedModel): return last_hidden_states -@add_start_docstrings( - """Transformer speech encoder consisting of *config.speech_encoder_layers* conformer self attention layers. - Each layer is a [`SeamlessM4TConformerEncoderLayer`].""", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer speech encoder consisting of *config.speech_encoder_layers* conformer self attention layers. + Each layer is a [`SeamlessM4TConformerEncoderLayer`]. + """ ) class SeamlessM4TSpeechEncoder(SeamlessM4TPreTrainedModel): main_input_name = "input_features" @@ -1625,15 +1522,10 @@ class SeamlessM4TSpeechEncoder(SeamlessM4TPreTrainedModel): # inspired from MBart and NllbMoe -@add_start_docstrings( - "Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`SeamlessM4TEncoderLayer`].", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`SeamlessM4TEncoderLayer`]. """ - embed_tokens (`nn.Embedding`, *optional*): - Input embedding - is_t2u_encoder (`bool`, *optional*, defaults to `False`): - indicates if it belongs to the text-to-units model, in which case it won't have input embeddings - """, ) class SeamlessM4TEncoder(SeamlessM4TPreTrainedModel): def __init__( @@ -1642,6 +1534,12 @@ class SeamlessM4TEncoder(SeamlessM4TPreTrainedModel): embed_tokens: Optional[nn.Embedding] = None, is_t2u_encoder: bool = False, ): + r""" + embed_tokens (`nn.Embedding`, *optional*): + Input embedding + is_t2u_encoder (`bool`, *optional*, defaults to `False`): + indicates if it belongs to the text-to-units model, in which case it won't have input embeddings + """ super().__init__(config) self.dropout = config.dropout @@ -1813,13 +1711,10 @@ class SeamlessM4TEncoder(SeamlessM4TPreTrainedModel): ) -@add_start_docstrings( - "Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SeamlessM4TDecoderLayer`].", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SeamlessM4TDecoderLayer`]. """ - embed_tokens (`nn.Embedding`, *optional*): - Input embedding - """, ) class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel): def __init__( @@ -1827,6 +1722,10 @@ class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel): config: SeamlessM4TConfig, embed_tokens: Optional[nn.Embedding] = None, ): + r""" + embed_tokens (`nn.Embedding`, *optional*): + Input embedding + """ super().__init__(config) self.dropout = config.dropout self.layerdrop = config.decoder_layerdrop @@ -1874,6 +1773,7 @@ class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1887,58 +1787,6 @@ class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: - r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - of the decoder. - encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -2061,12 +1909,10 @@ class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel): ) -@add_start_docstrings( - "Transformer bare text-to-unit encoder-decoder. The encoder is a [`SeamlessM4TEncoder`] without embeddings and the decoder is a [`SeamlessM4TDecoder`].", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer bare text-to-unit encoder-decoder. The encoder is a [`SeamlessM4TEncoder`] without embeddings and the decoder is a [`SeamlessM4TDecoder`]. """ - embed_tokens_decoder (`nn.Embedding`, *optional*): input embedding of the decoder. - """, ) class SeamlessM4TTextToUnitModel(SeamlessM4TPreTrainedModel): def __init__( @@ -2074,6 +1920,10 @@ class SeamlessM4TTextToUnitModel(SeamlessM4TPreTrainedModel): config: SeamlessM4TConfig, embed_tokens_decoder: Optional[nn.Embedding] = None, ): + r""" + embed_tokens_decoder (`nn.Embedding`, *optional*): + input embedding of the decoder. + """ super().__init__(config) self.encoder = SeamlessM4TEncoder(config, is_t2u_encoder=True) @@ -2150,12 +2000,10 @@ class SeamlessM4TTextToUnitModel(SeamlessM4TPreTrainedModel): ) -@add_start_docstrings( - "Transformer text-to-unit encoder-decoder with a language model head. The base encoder-decoder model is a [`SeamlessM4TTextToUnit`].", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer text-to-unit encoder-decoder with a language model head. The base encoder-decoder model is a [`SeamlessM4TTextToUnit`]. """ - embed_tokens_decoder (`nn.Embedding`, *optional*): input embedding of the decoder. - """, ) class SeamlessM4TTextToUnitForConditionalGeneration(SeamlessM4TPreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = [ @@ -2171,6 +2019,10 @@ class SeamlessM4TTextToUnitForConditionalGeneration(SeamlessM4TPreTrainedModel, config: SeamlessM4TConfig, embed_tokens_decoder: Optional[nn.Embedding] = None, ): + r""" + embed_tokens_decoder (`nn.Embedding`, *optional*): + input embedding of the decoder. + """ # update config - used principality for bos_token_id etc. config = copy.deepcopy(config) for param, val in config.to_dict().items(): @@ -2203,7 +2055,7 @@ class SeamlessM4TTextToUnitForConditionalGeneration(SeamlessM4TPreTrainedModel, def set_input_embeddings(self, value): self.model.decoder.embed_tokens = value - @add_start_docstrings_to_model_forward(M4T_TEXT_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_COMMON_CUSTOM_ARGS) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2220,6 +2072,12 @@ class SeamlessM4TTextToUnitForConditionalGeneration(SeamlessM4TPreTrainedModel, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -2292,23 +2150,6 @@ class SeamlessM4TTextToUnitForConditionalGeneration(SeamlessM4TPreTrainedModel, ############ VOCODER related code ################ -HIFIGAN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SeamlessM4TConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - # Copied from transformers.models.speecht5.modeling_speecht5.HifiGanResidualBlock class HifiGanResidualBlock(nn.Module): def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), leaky_relu_slope=0.1): @@ -2481,9 +2322,10 @@ class SeamlessM4THifiGan(nn.Module): return waveform -@add_start_docstrings( - """Code HiFi-GAN vocoder as described in this [repository](https://github.com/facebookresearch/speech-resynthesis).""", - HIFIGAN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Code HiFi-GAN vocoder as described in this [repository](https://github.com/facebookresearch/speech-resynthesis). + """ ) class SeamlessM4TCodeHifiGan(PreTrainedModel): config_class = SeamlessM4TConfig @@ -2645,9 +2487,10 @@ class SeamlessM4TCodeHifiGan(PreTrainedModel): ############ WHOLE MODEL related code ################ -@add_start_docstrings( - "The text-to-text SeamlessM4T Model transformer which can be used for T2TT.", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text-to-text SeamlessM4T Model transformer which can be used for T2TT. + """ ) class SeamlessM4TForTextToText(SeamlessM4TPreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["speech_encoder", "t2u_model", "vocoder"] @@ -2697,7 +2540,7 @@ class SeamlessM4TForTextToText(SeamlessM4TPreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_TEXT_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_COMMON_CUSTOM_ARGS) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2715,6 +2558,12 @@ class SeamlessM4TForTextToText(SeamlessM4TPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -2911,9 +2760,10 @@ class SeamlessM4TForTextToText(SeamlessM4TPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - "The speech-to-text SeamlessM4T Model transformer which can be used for S2TT.", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The speech-to-text SeamlessM4T Model transformer which can be used for S2TT. + """ ) class SeamlessM4TForSpeechToText(SeamlessM4TPreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["text_decoder", "t2u_model", "vocoder"] @@ -2958,7 +2808,7 @@ class SeamlessM4TForSpeechToText(SeamlessM4TPreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_SPEECH_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_COMMON_CUSTOM_ARGS) def forward( self, input_features: Optional[torch.LongTensor] = None, @@ -2976,6 +2826,12 @@ class SeamlessM4TForSpeechToText(SeamlessM4TPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -3181,9 +3037,10 @@ class SeamlessM4TForSpeechToText(SeamlessM4TPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - "The text-to-speech SeamlessM4T Model transformer which can be used for T2ST.", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text-to-speech SeamlessM4T Model transformer which can be used for T2ST. + """ ) class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["speech_encoder"] @@ -3236,7 +3093,7 @@ class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_TEXT_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_COMMON_CUSTOM_ARGS) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -3253,6 +3110,12 @@ class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -3511,9 +3374,10 @@ class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - "The speech-to-speech SeamlessM4T Model transformer which can be used for S2ST.", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The speech-to-speech SeamlessM4T Model transformer which can be used for S2ST. + """ ) class SeamlessM4TForSpeechToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["text_encoder"] @@ -3561,7 +3425,7 @@ class SeamlessM4TForSpeechToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_SPEECH_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_COMMON_CUSTOM_ARGS) def forward( self, input_features: Optional[torch.LongTensor] = None, @@ -3579,6 +3443,12 @@ class SeamlessM4TForSpeechToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -3851,13 +3721,10 @@ class SeamlessM4TForSpeechToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - "The original SeamlessM4T Model transformer which can be used for every tasks available (S2ST, S2TT, T2TT, T2ST).", - SEAMLESS_M4T_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The original SeamlessM4T Model transformer which can be used for every tasks available (S2ST, S2TT, T2TT, T2ST). """ - current_modality (`str`, *optional*, defaults to `"text"`): - Default modality. Used to initialize the model. - """, ) class SeamlessM4TModel(SeamlessM4TPreTrainedModel, GenerationMixin): _tied_weights_keys = [ @@ -3867,6 +3734,10 @@ class SeamlessM4TModel(SeamlessM4TPreTrainedModel, GenerationMixin): ] def __init__(self, config, current_modality="text"): + r""" + current_modality (`str`, *optional*, defaults to `"text"`): + Default modality. Used to initialize the model. + """ super().__init__(config) self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id) @@ -3923,7 +3794,7 @@ class SeamlessM4TModel(SeamlessM4TPreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_MODEL_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_COMMON_CUSTOM_ARGS) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -3942,6 +3813,12 @@ class SeamlessM4TModel(SeamlessM4TPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/seamless_m4t_v2/modeling_seamless_m4t_v2.py b/src/transformers/models/seamless_m4t_v2/modeling_seamless_m4t_v2.py index 704b75450f..69dc832967 100644 --- a/src/transformers/models/seamless_m4t_v2/modeling_seamless_m4t_v2.py +++ b/src/transformers/models/seamless_m4t_v2/modeling_seamless_m4t_v2.py @@ -37,19 +37,42 @@ from ...modeling_outputs import ( Wav2Vec2BaseModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_seamless_m4t_v2 import SeamlessM4Tv2Config logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "" -_CONFIG_FOR_DOC = "SeamlessM4Tv2Config" +SEAMLESS_M4T_V2_COMMON_CUSTOM_ARGS = r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_banks)`): + Input audio features. This should be returned by the [`SeamlessM4TFeatureExtractor`] class or the + [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + inputs_embeds (`torch.FloatTensor` of shape`(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. +""" @dataclass @@ -163,172 +186,6 @@ class SeamlessM4Tv2TextToUnitOutput(ModelOutput): loss: Optional[torch.FloatTensor] = None -SEAMLESS_M4T_V2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`~SeamlessM4Tv2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SEAMLESS_M4T_V2_MULTIMODAL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_banks)`): - Input audio features. This should be returned by the [`SeamlessM4TFeatureExtractor`] class or the - [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details. - """ - -M4T_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - """ - -M4T_SPEECH_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_banks)`): - Input audio features. This should be returned by the [`SeamlessM4TFeatureExtractor`] class or the - [`SeamlessM4TProcessor`] class. See [`SeamlessM4TFeatureExtractor.__call__`] for details. - """ - -SEAMLESS_M4T_V2_END_INPUTS_DOCSTRING = r""" - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape`(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the - loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -M4T_MODEL_INPUTS_DOCSTRING = SEAMLESS_M4T_V2_MULTIMODAL_INPUTS_DOCSTRING + SEAMLESS_M4T_V2_END_INPUTS_DOCSTRING - -M4T_TEXT_INPUTS_DOCSTRING = M4T_TEXT_INPUTS_DOCSTRING + SEAMLESS_M4T_V2_END_INPUTS_DOCSTRING - -M4T_SPEECH_INPUTS_DOCSTRING = M4T_SPEECH_INPUTS_DOCSTRING + SEAMLESS_M4T_V2_END_INPUTS_DOCSTRING - -M4T_TEXT_TO_UNITS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`SeamlessM4TTokenizer`] or [`SeamlessM4TProcessor`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - char_input_ids (`torch.LongTensor` of shape `(batch_size, char_sequence_length)`): - Character indices. The correspondence between characters and indices can be found in `char_to_id`, a - dictionary in the generation configuration. - char_count_per_id (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Number of characters per input id. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape`(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., - config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the - loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - ############ UTILS ################ @@ -1430,12 +1287,8 @@ class SeamlessM4Tv2TextToUnitDecoderLayer(nn.Module): ############ SUB-MODELS related code ################ +@auto_docstring class SeamlessM4Tv2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SeamlessM4Tv2Config base_model_prefix = "seamless_m4t_v2" supports_gradient_checkpointing = True @@ -1665,10 +1518,11 @@ class SeamlessM4Tv2PreTrainedModel(PreTrainedModel): return hidden_states -@add_start_docstrings( - """Transformer speech encoder consisting of *config.speech_encoder_layers* conformer self attention layers. - Each layer is a [`SeamlessM4Tv2ConformerEncoderLayer`].""", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer speech encoder consisting of *config.speech_encoder_layers* conformer self attention layers. + Each layer is a [`SeamlessM4Tv2ConformerEncoderLayer`]. + """ ) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TSpeechEncoder with SeamlessM4T->SeamlessM4Tv2 class SeamlessM4Tv2SpeechEncoder(SeamlessM4Tv2PreTrainedModel): @@ -1738,15 +1592,10 @@ class SeamlessM4Tv2SpeechEncoder(SeamlessM4Tv2PreTrainedModel): # inspired from MBart and NllbMoe -@add_start_docstrings( - "Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`SeamlessM4Tv2EncoderLayer`].", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`SeamlessM4Tv2EncoderLayer`]. """ - embed_tokens (`nn.Embedding`, *optional*): - Input embedding - is_t2u_encoder (`bool`, *optional*, defaults to `False`): - indicates if it belongs to the text-to-units model, in which case it won't have input embeddings - """, ) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TEncoder with SeamlessM4T->SeamlessM4Tv2 class SeamlessM4Tv2Encoder(SeamlessM4Tv2PreTrainedModel): @@ -1756,6 +1605,12 @@ class SeamlessM4Tv2Encoder(SeamlessM4Tv2PreTrainedModel): embed_tokens: Optional[nn.Embedding] = None, is_t2u_encoder: bool = False, ): + r""" + embed_tokens (`nn.Embedding`, *optional*): + Input embedding + is_t2u_encoder (`bool`, *optional*, defaults to `False`): + indicates if it belongs to the text-to-units model, in which case it won't have input embeddings + """ super().__init__(config) self.dropout = config.dropout @@ -1927,13 +1782,10 @@ class SeamlessM4Tv2Encoder(SeamlessM4Tv2PreTrainedModel): ) -@add_start_docstrings( - "Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SeamlessM4Tv2DecoderLayer`].", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SeamlessM4Tv2DecoderLayer`]. """ - embed_tokens (`nn.Embedding`, *optional*): - Input embedding - """, ) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TDecoder with SeamlessM4T->SeamlessM4Tv2 class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel): @@ -1942,6 +1794,10 @@ class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel): config: SeamlessM4Tv2Config, embed_tokens: Optional[nn.Embedding] = None, ): + r""" + embed_tokens (`nn.Embedding`, *optional*): + Input embedding + """ super().__init__(config) self.dropout = config.dropout self.layerdrop = config.decoder_layerdrop @@ -1989,6 +1845,7 @@ class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2002,58 +1859,6 @@ class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: - r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - of the decoder. - encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -2176,13 +1981,10 @@ class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel): ) -@add_start_docstrings( - "Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SeamlessM4Tv2DecoderLayer`].", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SeamlessM4Tv2DecoderLayer`]. """ - embed_tokens (`nn.Embedding`, *optional*): - Input embedding - """, ) class SeamlessM4Tv2TextToUnitDecoder(SeamlessM4Tv2PreTrainedModel): def __init__( @@ -2190,6 +1992,10 @@ class SeamlessM4Tv2TextToUnitDecoder(SeamlessM4Tv2PreTrainedModel): config: SeamlessM4Tv2Config, embed_tokens: Optional[nn.Embedding] = None, ): + r""" + embed_tokens (`nn.Embedding`, *optional*): + Input embedding + """ super().__init__(config) self.dropout = config.dropout self.layerdrop = config.decoder_layerdrop @@ -2358,12 +2164,10 @@ class SeamlessM4Tv2TextToUnitDecoder(SeamlessM4Tv2PreTrainedModel): ) -@add_start_docstrings( - "Transformer bare text-to-unit encoder-decoder. The encoder is a [`SeamlessM4Tv2Encoder`] without embeddings and the decoder is a [`SeamlessM4Tv2TextToUnitDecoder`].", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer bare text-to-unit encoder-decoder. The encoder is a [`SeamlessM4Tv2Encoder`] without embeddings and the decoder is a [`SeamlessM4Tv2TextToUnitDecoder`]. """ - embed_tokens_decoder (`nn.Embedding`, *optional*): input embedding of the decoder. - """, ) class SeamlessM4Tv2TextToUnitModel(SeamlessM4Tv2PreTrainedModel): # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TTextToUnitModel.__init__ with SeamlessM4T->SeamlessM4Tv2, Decoder->TextToUnitDecoder @@ -2372,6 +2176,10 @@ class SeamlessM4Tv2TextToUnitModel(SeamlessM4Tv2PreTrainedModel): config: SeamlessM4Tv2Config, embed_tokens_decoder: Optional[nn.Embedding] = None, ): + r""" + embed_tokens_decoder (`nn.Embedding`, *optional*): + input embedding of the decoder. + """ super().__init__(config) self.encoder = SeamlessM4Tv2Encoder(config, is_t2u_encoder=True) @@ -2439,12 +2247,10 @@ class SeamlessM4Tv2TextToUnitModel(SeamlessM4Tv2PreTrainedModel): ) -@add_start_docstrings( - "Transformer text-to-unit encoder-decoder with a language model head. The base encoder-decoder model is a [`SeamlessM4Tv2TextToUnitModel`].", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Transformer text-to-unit encoder-decoder with a language model head. The base encoder-decoder model is a [`SeamlessM4Tv2TextToUnitModel`]. """ - embed_tokens_decoder (`nn.Embedding`, *optional*): input embedding of the decoder. - """, ) class SeamlessM4Tv2TextToUnitForConditionalGeneration(SeamlessM4Tv2PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = [ @@ -2461,6 +2267,10 @@ class SeamlessM4Tv2TextToUnitForConditionalGeneration(SeamlessM4Tv2PreTrainedMod config: SeamlessM4Tv2Config, embed_tokens_decoder: Optional[nn.Embedding] = None, ): + r""" + embed_tokens_decoder (`nn.Embedding`, *optional*): + input embedding of the decoder. + """ # update config - used principality for bos_token_id etc. config = copy.deepcopy(config) for param, val in config.to_dict().items(): @@ -2499,7 +2309,7 @@ class SeamlessM4Tv2TextToUnitForConditionalGeneration(SeamlessM4Tv2PreTrainedMod def set_input_embeddings(self, value): self.model.decoder.embed_tokens = value - @add_start_docstrings_to_model_forward(M4T_TEXT_TO_UNITS_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2514,6 +2324,21 @@ class SeamlessM4Tv2TextToUnitForConditionalGeneration(SeamlessM4Tv2PreTrainedMod return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + char_input_ids (`torch.LongTensor` of shape `(batch_size, char_sequence_length)`): + Character indices. The correspondence between characters and indices can be found in `char_to_id`, a + dictionary in the generation configuration. + char_count_per_id (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Number of characters per input id. + inputs_embeds (`torch.FloatTensor` of shape`(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.model( @@ -2561,23 +2386,6 @@ class SeamlessM4Tv2TextToUnitForConditionalGeneration(SeamlessM4Tv2PreTrainedMod ############ VOCODER related code ################ -HIFIGAN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SeamlessM4Tv2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - # Copied from transformers.models.speecht5.modeling_speecht5.HifiGanResidualBlock class HifiGanResidualBlock(nn.Module): def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), leaky_relu_slope=0.1): @@ -2751,9 +2559,10 @@ class SeamlessM4Tv2HifiGan(nn.Module): return waveform -@add_start_docstrings( - """Code HiFi-GAN vocoder as described in this [repository](https://github.com/facebookresearch/speech-resynthesis).""", - HIFIGAN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Code HiFi-GAN vocoder as described in this [repository](https://github.com/facebookresearch/speech-resynthesis). + """ ) class SeamlessM4Tv2CodeHifiGan(PreTrainedModel): config_class = SeamlessM4Tv2Config @@ -2924,11 +2733,12 @@ class SeamlessM4Tv2CodeHifiGan(PreTrainedModel): ############ WHOLE MODEL related code ################ -@add_start_docstrings( - "The text-to-text SeamlessM4Tv2 Model transformer which can be used for T2TT.", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text-to-text SeamlessM4Tv2 Model transformer which can be used for T2TT. + """ ) -# Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToText with SeamlessM4T->SeamlessM4Tv2,SeamlessM4Tv2Tokenizer->SeamlessM4TTokenizer, SeamlessM4Tv2Processor->SeamlessM4TProcessor +# Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToText with SeamlessM4T->SeamlessM4Tv2,SeamlessM4Tv2Tokenizer->SeamlessM4TTokenizer, SeamlessM4Tv2Processor->SeamlessM4TProcessor, SEAMLESS_M4T->SEAMLESS_M4T_V2 class SeamlessM4Tv2ForTextToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["speech_encoder", "t2u_model", "vocoder"] main_input_name = "input_ids" @@ -2977,7 +2787,7 @@ class SeamlessM4Tv2ForTextToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_TEXT_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_V2_COMMON_CUSTOM_ARGS) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2995,6 +2805,12 @@ class SeamlessM4Tv2ForTextToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -3191,9 +3007,10 @@ class SeamlessM4Tv2ForTextToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - "The speech-to-text SeamlessM4Tv2 Model transformer which can be used for S2TT.", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The speech-to-text SeamlessM4Tv2 Model transformer which can be used for S2TT. + """ ) class SeamlessM4Tv2ForSpeechToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["text_decoder", "t2u_model", "vocoder"] @@ -3246,7 +3063,7 @@ class SeamlessM4Tv2ForSpeechToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_SPEECH_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_V2_COMMON_CUSTOM_ARGS) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToText.forward def forward( self, @@ -3265,6 +3082,12 @@ class SeamlessM4Tv2ForSpeechToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -3472,9 +3295,10 @@ class SeamlessM4Tv2ForSpeechToText(SeamlessM4Tv2PreTrainedModel, GenerationMixin return reordered_past -@add_start_docstrings( - "The text-to-speech SeamlessM4Tv2 Model transformer which can be used for T2ST.", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text-to-speech SeamlessM4Tv2 Model transformer which can be used for T2ST. + """ ) class SeamlessM4Tv2ForTextToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["speech_encoder"] @@ -3535,7 +3359,7 @@ class SeamlessM4Tv2ForTextToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMixin self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_TEXT_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_V2_COMMON_CUSTOM_ARGS) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForTextToSpeech.forward with SeamlessM4T->SeamlessM4Tv2 def forward( self, @@ -3553,6 +3377,12 @@ class SeamlessM4Tv2ForTextToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMixin output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -3843,9 +3673,10 @@ class SeamlessM4Tv2ForTextToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMixin return reordered_past -@add_start_docstrings( - "The speech-to-speech SeamlessM4Tv2 Model transformer which can be used for S2ST.", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The speech-to-speech SeamlessM4Tv2 Model transformer which can be used for S2ST. + """ ) class SeamlessM4Tv2ForSpeechToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_missing = ["text_encoder"] @@ -3901,7 +3732,7 @@ class SeamlessM4Tv2ForSpeechToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMix self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_SPEECH_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_V2_COMMON_CUSTOM_ARGS) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TForSpeechToSpeech.forward with SeamlessM4T->SeamlessM4Tv2 def forward( self, @@ -3920,6 +3751,12 @@ class SeamlessM4Tv2ForSpeechToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMix return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ if labels is not None: if use_cache: logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") @@ -4223,14 +4060,10 @@ class SeamlessM4Tv2ForSpeechToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMix return reordered_past -@add_start_docstrings( - "The original SeamlessM4Tv2 Model transformer which can be used for every tasks available (S2ST, S2TT, T2TT, T2ST).", - SEAMLESS_M4T_V2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The original SeamlessM4Tv2 Model transformer which can be used for every tasks available (S2ST, S2TT, T2TT, T2ST). """ - current_modality (`str`, *optional*, defaults to `"text"`): - Default modality. Used only to initialize the model. It can be set to `"text"` or `"speech"`. - This will be updated automatically according to the modality passed to the forward and generate passes (`input_ids` for text and `input_features` for audio). - """, ) class SeamlessM4Tv2Model(SeamlessM4Tv2PreTrainedModel, GenerationMixin): _tied_weights_keys = [ @@ -4241,6 +4074,10 @@ class SeamlessM4Tv2Model(SeamlessM4Tv2PreTrainedModel, GenerationMixin): # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.__init__ with SeamlessM4T->SeamlessM4Tv2 def __init__(self, config, current_modality="text"): + r""" + current_modality (`str`, *optional*, defaults to `"text"`): + Default modality. Used to initialize the model. + """ super().__init__(config) self.shared = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id) @@ -4304,7 +4141,7 @@ class SeamlessM4Tv2Model(SeamlessM4Tv2PreTrainedModel, GenerationMixin): self._tie_or_clone_weights(self.text_decoder.embed_tokens, self.shared) self._tie_or_clone_weights(self.lm_head, self.shared) - @add_start_docstrings_to_model_forward(M4T_MODEL_INPUTS_DOCSTRING) + @auto_docstring(custom_args=SEAMLESS_M4T_V2_COMMON_CUSTOM_ARGS) # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TModel.forward with SeamlessM4T->SeamlessM4Tv2 def forward( self, @@ -4324,6 +4161,12 @@ class SeamlessM4Tv2Model(SeamlessM4Tv2PreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, **kwargs, ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/segformer/modeling_segformer.py b/src/transformers/models/segformer/modeling_segformer.py index 4f9da2cab7..81df26dc69 100755 --- a/src/transformers/models/segformer/modeling_segformer.py +++ b/src/transformers/models/segformer/modeling_segformer.py @@ -26,31 +26,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, ImageClassifierOutput, SemanticSegmenterOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_segformer import SegformerConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "SegformerConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "nvidia/mit-b0" -_EXPECTED_OUTPUT_SHAPE = [1, 256, 16, 16] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "nvidia/mit-b0" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - - class SegFormerImageClassifierOutput(ImageClassifierOutput): """ Base class for outputs of image classification models. @@ -443,12 +425,8 @@ class SegformerEncoder(nn.Module): ) +@auto_docstring class SegformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SegformerConfig base_model_prefix = "segformer" main_input_name = "pixel_values" @@ -470,39 +448,7 @@ class SegformerPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -SEGFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SegformerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SEGFORMER_INPUTS_DOCSTRING = r""" - - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SegFormer encoder (Mix-Transformer) outputting raw hidden-states without any specific head on top.", - SEGFORMER_START_DOCSTRING, -) +@auto_docstring class SegformerModel(SegformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -522,14 +468,7 @@ class SegformerModel(SegformerPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SEGFORMER_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -561,12 +500,11 @@ class SegformerModel(SegformerPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" SegFormer Model transformer with an image classification head on top (a linear layer on top of the final hidden states) e.g. for ImageNet. - """, - SEGFORMER_START_DOCSTRING, + """ ) class SegformerForImageClassification(SegformerPreTrainedModel): def __init__(self, config): @@ -581,13 +519,7 @@ class SegformerForImageClassification(SegformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SEGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=SegFormerImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -732,9 +664,10 @@ class SegformerDecodeHead(SegformerPreTrainedModel): return logits -@add_start_docstrings( - """SegFormer Model transformer with an all-MLP decode head on top e.g. for ADE20k, CityScapes.""", - SEGFORMER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SegFormer Model transformer with an all-MLP decode head on top e.g. for ADE20k, CityScapes. + """ ) class SegformerForSemanticSegmentation(SegformerPreTrainedModel): def __init__(self, config): @@ -745,8 +678,7 @@ class SegformerForSemanticSegmentation(SegformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SEGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -760,8 +692,6 @@ class SegformerForSemanticSegmentation(SegformerPreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/seggpt/modeling_seggpt.py b/src/transformers/models/seggpt/modeling_seggpt.py index 0150f2d7a9..ff4a908452 100644 --- a/src/transformers/models/seggpt/modeling_seggpt.py +++ b/src/transformers/models/seggpt/modeling_seggpt.py @@ -25,26 +25,12 @@ from torch.nn import functional as F from ...activations import ACT2FN from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_seggpt import SegGptConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "SegGptConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "BAAI/seggpt-vit-large" -_EXPECTED_OUTPUT_SHAPE = [3, 896, 448] - @dataclass class SegGptEncoderOutput(ModelOutput): @@ -613,12 +599,8 @@ class SegGptDecoder(nn.Module): return hidden_states +@auto_docstring class SegGptPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SegGptConfig base_model_prefix = "model" main_input_name = "pixel_values" @@ -666,58 +648,7 @@ class SegGptPreTrainedModel(PreTrainedModel): torch.nn.init.normal_(module.type_token_instance, std=std) -SEGGPT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SegGptConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SEGGPT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`SegGptImageProcessor.__call__`] - for details. - - prompt_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Prompt pixel values. Prompt pixel values can be obtained using [`AutoImageProcessor`]. See - [`SegGptImageProcessor.__call__`] for details. - - prompt_masks (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Prompt mask. Prompt mask can be obtained using [`AutoImageProcessor`]. See [`SegGptImageProcessor.__call__`] for - details. - - bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - - feature_ensemble (`bool`, *optional*): - Boolean indicating whether to use feature ensemble or not. If `True`, the model will use feature ensemble - if we have at least two prompts. If `False`, the model will not use feature ensemble. This argument should - be considered when doing few-shot inference on an input image i.e. more than one prompt for the same image. - - embedding_type (`str`, *optional*): - Embedding type. Indicates whether the prompt is a semantic or instance embedding. Can be either - instance or semantic. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SegGpt Model transformer outputting raw hidden-states without any specific head on top.", - SEGGPT_START_DOCSTRING, -) +@auto_docstring class SegGptModel(SegGptPreTrainedModel): def __init__(self, config: SegGptConfig): super().__init__(config) @@ -740,8 +671,7 @@ class SegGptModel(SegGptPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SEGGPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SegGptEncoderOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -756,11 +686,24 @@ class SegGptModel(SegGptPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SegGptEncoderOutput]: r""" + prompt_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Prompt pixel values. Prompt pixel values can be obtained using [`AutoImageProcessor`]. See + [`SegGptImageProcessor.__call__`] for details. + prompt_masks (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Prompt mask. Prompt mask can be obtained using [`AutoImageProcessor`]. See [`SegGptImageProcessor.__call__`] for + details. + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + feature_ensemble (`bool`, *optional*): + Boolean indicating whether to use feature ensemble or not. If `True`, the model will use feature ensemble + if we have at least two prompts. If `False`, the model will not use feature ensemble. This argument should + be considered when doing few-shot inference on an input image i.e. more than one prompt for the same image. + embedding_type (`str`, *optional*): + Embedding type. Indicates whether the prompt is a semantic or instance embedding. Can be either + instance or semantic. labels (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`, `optional`): Ground truth mask for input images. - Returns: - Examples: ```python @@ -908,9 +851,10 @@ class SegGptLoss(nn.Module): return loss -@add_start_docstrings( - "SegGpt model with a decoder on top for one-shot image segmentation.", - SEGGPT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SegGpt model with a decoder on top for one-shot image segmentation. + """ ) class SegGptForImageSegmentation(SegGptPreTrainedModel): def __init__(self, config: SegGptConfig): @@ -923,8 +867,7 @@ class SegGptForImageSegmentation(SegGptPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SEGGPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SegGptImageSegmentationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -939,11 +882,24 @@ class SegGptForImageSegmentation(SegGptPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SegGptImageSegmentationOutput]: r""" + prompt_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Prompt pixel values. Prompt pixel values can be obtained using [`AutoImageProcessor`]. See + [`SegGptImageProcessor.__call__`] for details. + prompt_masks (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Prompt mask. Prompt mask can be obtained using [`AutoImageProcessor`]. See [`SegGptImageProcessor.__call__`] for + details. + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + feature_ensemble (`bool`, *optional*): + Boolean indicating whether to use feature ensemble or not. If `True`, the model will use feature ensemble + if we have at least two prompts. If `False`, the model will not use feature ensemble. This argument should + be considered when doing few-shot inference on an input image i.e. more than one prompt for the same image. + embedding_type (`str`, *optional*): + Embedding type. Indicates whether the prompt is a semantic or instance embedding. Can be either + instance or semantic. labels (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`, `optional`): Ground truth mask for input images. - Returns: - Examples: ```python diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py index 572c07e3c9..8bfeca060b 100644 --- a/src/transformers/models/sew/modeling_sew.py +++ b/src/transformers/models/sew/modeling_sew.py @@ -30,12 +30,7 @@ from ...integrations.fsdp import is_fsdp_managed_module from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import auto_docstring, logging from .configuration_sew import SEWConfig @@ -48,24 +43,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 1 -# General docstring -_CONFIG_FOR_DOC = "SEWConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "asapp/sew-tiny-100k-ft-ls100h" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 512] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = ( - "'MISTER QUILTER IS THE APPOSTILE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPOLLE'" -) -_CTC_EXPECTED_LOSS = 0.42 - -# Audio class docstring -_SEQ_CLASS_CHECKPOINT = "anton-l/sew-mid-100k-ft-keyword-spotting" -_SEQ_CLASS_EXPECTED_OUTPUT = "'_unknown_'" -_SEQ_CLASS_EXPECTED_LOSS = 9.52 - # Copied from transformers.models.wav2vec2.modeling_wav2vec2._compute_mask_indices def _compute_mask_indices( @@ -967,12 +944,8 @@ class SEWEncoder(nn.Module): ) +@auto_docstring class SEWPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SEWConfig base_model_prefix = "sew" main_input_name = "input_values" @@ -1040,56 +1013,7 @@ class SEWPreTrainedModel(PreTrainedModel): return attention_mask -SEW_START_DOCSTRING = r""" - SEW was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech - Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, - Yoav Artzi. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SEWConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -SEW_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SEW Model transformer outputting raw hidden-states without any specific head on top.", - SEW_START_DOCSTRING, -) +@auto_docstring class SEWModel(SEWPreTrainedModel): def __init__(self, config: SEWConfig): super().__init__(config) @@ -1157,14 +1081,7 @@ class SEWModel(SEWPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(SEW_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1174,6 +1091,11 @@ class SEWModel(SEWPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1214,13 +1136,20 @@ class SEWModel(SEWPreTrainedModel): ) -@add_start_docstrings( - """SEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - SEW_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->SEW, wav2vec2->sew, WAV2VEC2->SEW class SEWForCTC(SEWPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`SEWForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.sew = SEWModel(config) @@ -1291,14 +1220,7 @@ class SEWForCTC(SEWPreTrainedModel): for param in self.sew.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(SEW_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1370,12 +1292,11 @@ class SEWForCTC(SEWPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" SEW Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - SEW_START_DOCSTRING, + """ ) # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForSequenceClassification with Wav2Vec2->SEW, wav2vec2->sew, WAV2VEC2->SEW class SEWForSequenceClassification(SEWPreTrainedModel): @@ -1423,15 +1344,7 @@ class SEWForSequenceClassification(SEWPreTrainedModel): for param in self.sew.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(SEW_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_SEQ_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1442,6 +1355,11 @@ class SEWForSequenceClassification(SEWPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`SEWProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py index 96c587fbb2..27ab17799a 100644 --- a/src/transformers/models/sew_d/modeling_sew_d.py +++ b/src/transformers/models/sew_d/modeling_sew_d.py @@ -30,7 +30,7 @@ from ...integrations.deepspeed import is_deepspeed_zero3_enabled from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import softmax_backward_data -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_sew_d import SEWDConfig @@ -39,23 +39,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 1 -# General docstring -_CONFIG_FOR_DOC = "SEWDConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "asapp/sew-d-tiny-100k-ft-ls100h" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 384] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTIL OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 0.21 - -# Audio class docstring -_SEQ_CLASS_CHECKPOINT = "anton-l/sew-d-mid-400k-ft-keyword-spotting" -_SEQ_CLASS_EXPECTED_OUTPUT = "'_unknown_'" -_SEQ_CLASS_EXPECTED_LOSS = 3.16 - - # Copied from transformers.models.wav2vec2.modeling_wav2vec2._compute_mask_indices def _compute_mask_indices( shape: Tuple[int, int], @@ -1214,12 +1197,8 @@ class SEWDEncoder(nn.Module): ) +@auto_docstring class SEWDPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SEWDConfig base_model_prefix = "sew-d" main_input_name = "input_values" @@ -1289,56 +1268,7 @@ class SEWDPreTrainedModel(PreTrainedModel): return attention_mask -SEWD_START_DOCSTRING = r""" - SEW-D was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech - Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, - Yoav Artzi. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SEWDConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -SEWD_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SEW-D Model transformer outputting raw hidden-states without any specific head on top.", - SEWD_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.sew.modeling_sew.SEWModel with SEW->SEWD, layer_norm_eps->feature_layer_norm_eps class SEWDModel(SEWDPreTrainedModel): def __init__(self, config: SEWDConfig): @@ -1407,14 +1337,7 @@ class SEWDModel(SEWDPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(SEWD_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1424,6 +1347,11 @@ class SEWDModel(SEWDPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1464,13 +1392,20 @@ class SEWDModel(SEWDPreTrainedModel): ) -@add_start_docstrings( - """SEW-D Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - SEWD_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SEW-D Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->SEWD, wav2vec2->sew_d, WAV2VEC2->SEWD class SEWDForCTC(SEWDPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`SEWDForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.sew_d = SEWDModel(config) @@ -1541,14 +1476,7 @@ class SEWDForCTC(SEWDPreTrainedModel): for param in self.sew_d.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(SEWD_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1620,12 +1548,11 @@ class SEWDForCTC(SEWDPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" SEWD Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - SEWD_START_DOCSTRING, + """ ) # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForSequenceClassification with Wav2Vec2->SEWD, wav2vec2->sew_d, WAV2VEC2->SEWD class SEWDForSequenceClassification(SEWDPreTrainedModel): @@ -1673,15 +1600,7 @@ class SEWDForSequenceClassification(SEWDPreTrainedModel): for param in self.sew_d.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(SEWD_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_SEQ_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1692,6 +1611,11 @@ class SEWDForSequenceClassification(SEWDPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`SEWDProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/shieldgemma2/modeling_shieldgemma2.py b/src/transformers/models/shieldgemma2/modeling_shieldgemma2.py index a0a65ea37c..a9b897e2df 100644 --- a/src/transformers/models/shieldgemma2/modeling_shieldgemma2.py +++ b/src/transformers/models/shieldgemma2/modeling_shieldgemma2.py @@ -23,92 +23,15 @@ from ...cache_utils import Cache from ...modeling_outputs import ImageClassifierOutputWithNoAttention from ...modeling_utils import PreTrainedModel from ...utils import ( - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) from ..auto import AutoModelForImageTextToText from .configuration_shieldgemma2 import ShieldGemma2Config -_CHECKPOINT_FOR_DOC = "google/shieldgemma-2-4b-it" -_CONFIG_FOR_DOC = "ShieldGemma2Config" - logger = logging.get_logger(__name__) -SHIELDGEMMA2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - @dataclass class ShieldGemma2ImageClassifierOutputWithNoAttention(ImageClassifierOutputWithNoAttention): @@ -119,6 +42,7 @@ class ShieldGemma2ImageClassifierOutputWithNoAttention(ImageClassifierOutputWith probabilities: Optional[torch.Tensor] = None +@auto_docstring class ShieldGemma2ForImageClassification(PreTrainedModel): config_class = ShieldGemma2Config @@ -149,7 +73,7 @@ class ShieldGemma2ForImageClassification(PreTrainedModel): def tie_weights(self): return self.model.language_model.tie_weights() - @add_start_docstrings_to_model_forward(SHIELDGEMMA2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -168,8 +92,7 @@ class ShieldGemma2ForImageClassification(PreTrainedModel): logits_to_keep: Union[int, torch.Tensor] = 0, **lm_kwargs, ) -> ShieldGemma2ImageClassifierOutputWithNoAttention: - """Predicts the binary probability that the image violates the specified policy. - + r""" Returns: A `ShieldGemma2ImageClassifierOutputWithNoAttention` instance containing the logits and probabilities associated with the model predicting the `Yes` or `No` token as the response to that prompt, captured in the diff --git a/src/transformers/models/siglip/image_processing_siglip_fast.py b/src/transformers/models/siglip/image_processing_siglip_fast.py index b28f89dbbf..de2426e6eb 100644 --- a/src/transformers/models/siglip/image_processing_siglip_fast.py +++ b/src/transformers/models/siglip/image_processing_siglip_fast.py @@ -14,19 +14,16 @@ # limitations under the License. """Fast Image processor class for SigLIP.""" -from ...image_processing_utils_fast import BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast +from ...image_processing_utils_fast import BaseImageProcessorFast from ...image_utils import ( IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling, ) -from ...utils import add_start_docstrings +from ...utils import auto_docstring -@add_start_docstrings( - "Constructs a fast SigLIP image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class SiglipImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BICUBIC image_mean = IMAGENET_STANDARD_MEAN diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index f77248970c..252315d418 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -31,24 +31,12 @@ from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging, torch_int from .configuration_siglip import SiglipConfig, SiglipTextConfig, SiglipVisionConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "SiglipConfig" -_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224" - def _trunc_normal_(tensor, mean, std, a, b): # Cut & paste from PyTorch official master until it's in a few official releases - RW @@ -514,12 +502,8 @@ class SiglipEncoderLayer(GradientCheckpointingLayer): return outputs +@auto_docstring class SiglipPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SiglipConfig base_model_prefix = "siglip" supports_gradient_checkpointing = True @@ -581,110 +565,6 @@ class SiglipPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -SIGLIP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SiglipConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SIGLIP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -SIGLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -SIGLIP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->Siglip class SiglipEncoder(nn.Module): """ @@ -779,8 +659,7 @@ class SiglipTextTransformer(nn.Module): self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -789,10 +668,6 @@ class SiglipTextTransformer(nn.Module): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -834,9 +709,10 @@ class SiglipTextTransformer(nn.Module): ) -@add_start_docstrings( - """The text model from SigLIP without any head or projection on top.""", - SIGLIP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text model from SigLIP without any head or projection on top. + """ ) class SiglipTextModel(SiglipPreTrainedModel): config_class = SiglipTextConfig @@ -854,8 +730,7 @@ class SiglipTextModel(SiglipPreTrainedModel): self.text_model.embeddings.token_embedding = value @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -865,8 +740,6 @@ class SiglipTextModel(SiglipPreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: r""" - Returns: - Examples: ```python @@ -906,8 +779,7 @@ class SiglipVisionTransformer(nn.Module): self.head = SiglipMultiheadAttentionPoolingHead(config) @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig) + @auto_docstring def forward( self, pixel_values, @@ -915,10 +787,6 @@ class SiglipVisionTransformer(nn.Module): output_hidden_states: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = False, ) -> BaseModelOutputWithPooling: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -969,9 +837,10 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module): return hidden_state[:, 0] -@add_start_docstrings( - """The vision model from SigLIP without any head or projection on top.""", - SIGLIP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from SigLIP without any head or projection on top. + """ ) class SiglipVisionModel(SiglipPreTrainedModel): config_class = SiglipVisionConfig @@ -989,8 +858,7 @@ class SiglipVisionModel(SiglipPreTrainedModel): return self.vision_model.embeddings.patch_embedding @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig) + @auto_docstring def forward( self, pixel_values, @@ -999,8 +867,6 @@ class SiglipVisionModel(SiglipPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> BaseModelOutputWithPooling: r""" - Returns: - Examples: ```python @@ -1029,7 +895,7 @@ class SiglipVisionModel(SiglipPreTrainedModel): ) -@add_start_docstrings(SIGLIP_START_DOCSTRING) +@auto_docstring class SiglipModel(SiglipPreTrainedModel): config_class = SiglipConfig @@ -1065,7 +931,7 @@ class SiglipModel(SiglipPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1111,7 +977,7 @@ class SiglipModel(SiglipPreTrainedModel): return pooled_output - @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1161,8 +1027,7 @@ class SiglipModel(SiglipPreTrainedModel): return pooled_output @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SiglipOutput, config_class=SiglipConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1175,7 +1040,8 @@ class SiglipModel(SiglipPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> SiglipOutput: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: @@ -1259,12 +1125,11 @@ class SiglipModel(SiglipPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" SigLIP vision encoder with an image classification head on top (a linear layer on top of the pooled final hidden states of the patch tokens) e.g. for ImageNet. - """, - SIGLIP_START_DOCSTRING, + """ ) class SiglipForImageClassification(SiglipPreTrainedModel): main_input_name = "pixel_values" @@ -1288,8 +1153,7 @@ class SiglipForImageClassification(SiglipPreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1304,8 +1168,6 @@ class SiglipForImageClassification(SiglipPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/siglip2/image_processing_siglip2_fast.py b/src/transformers/models/siglip2/image_processing_siglip2_fast.py index dcd7cef629..191de2f961 100644 --- a/src/transformers/models/siglip2/image_processing_siglip2_fast.py +++ b/src/transformers/models/siglip2/image_processing_siglip2_fast.py @@ -20,8 +20,6 @@ import torch from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, @@ -33,7 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -86,21 +84,19 @@ def pad_along_first_dim( class Siglip2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + patch_size (`int`, *optional*, defaults to 16): + The size (resolution) of each patch the image will be split to. + max_num_patches (`int`, *optional*, defaults to 256): + The image will be resized to have at most this number of patches, + and then padded in "patch" dimension to match this number exactly. + """ + patch_size: Optional[int] max_num_patches: Optional[int] -@add_start_docstrings( - r"Constructs a fast Siglip2 image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - patch_size (`int`, *optional*, defaults to 16): - The size (resolution) of each patch the image will be split to. - max_num_patches (`int`, *optional*, defaults to 256): - The image will be resized to have at most this number of patches, - and then padded in "patch" dimension to match this number exactly. - """, -) +@auto_docstring class Siglip2ImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = [0.5, 0.5, 0.5] @@ -121,16 +117,7 @@ class Siglip2ImageProcessorFast(BaseImageProcessorFast): kwargs.pop("do_resize", None) return super()._validate_preprocess_kwargs(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - patch_size (`int`, *optional*, defaults to `self.patch_size`): - The size (resolution) of each patch the image will be split to. - max_num_patches (`int`, *optional*, defaults to `self.max_num_patches`): - The image will be resized to have at most this number of patches, - and then padded in "patch" dimension to match this number exactly. - """, - ) + @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[Siglip2FastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/siglip2/modeling_siglip2.py b/src/transformers/models/siglip2/modeling_siglip2.py index 0999ff6984..8614384241 100644 --- a/src/transformers/models/siglip2/modeling_siglip2.py +++ b/src/transformers/models/siglip2/modeling_siglip2.py @@ -35,22 +35,12 @@ from ...modeling_attn_mask_utils import _prepare_4d_attention_mask from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging from .configuration_siglip2 import Siglip2Config, Siglip2TextConfig, Siglip2VisionConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "Siglip2Config" - @dataclass class Siglip2VisionOutput(ModelOutput): @@ -485,24 +475,6 @@ class Siglip2Encoder(nn.Module): ) -SIGLIP2_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class Siglip2VisionTransformer(nn.Module): def __init__(self, config: Siglip2VisionConfig): super().__init__() @@ -518,8 +490,7 @@ class Siglip2VisionTransformer(nn.Module): self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Siglip2VisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -529,8 +500,8 @@ class Siglip2VisionTransformer(nn.Module): output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: r""" - Returns: - + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -700,39 +671,6 @@ def default_flax_embed_init(tensor): variance_scaling_(tensor, mode="fan_in", distribution="normal") -SIGLIP2_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class Siglip2TextTransformer(nn.Module): def __init__(self, config: Siglip2TextConfig): super().__init__() @@ -746,8 +684,7 @@ class Siglip2TextTransformer(nn.Module): self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP2_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Siglip2TextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -756,10 +693,6 @@ class Siglip2TextTransformer(nn.Module): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -801,67 +734,8 @@ class Siglip2TextTransformer(nn.Module): ) -SIGLIP2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Siglip2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SIGLIP2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - +@auto_docstring class Siglip2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Siglip2Config base_model_prefix = "siglip2" supports_gradient_checkpointing = True @@ -923,9 +797,10 @@ class Siglip2PreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -@add_start_docstrings( - """The text model from Siglip2 without any head or projection on top.""", - SIGLIP2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The text model from Siglip2 without any head or projection on top. + """ ) class Siglip2TextModel(Siglip2PreTrainedModel): config_class = Siglip2TextConfig @@ -943,8 +818,7 @@ class Siglip2TextModel(Siglip2PreTrainedModel): self.text_model.embeddings.token_embedding = value @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP2_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Siglip2TextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -954,8 +828,6 @@ class Siglip2TextModel(Siglip2PreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: r""" - Returns: - Examples: ```python @@ -1012,9 +884,10 @@ class Siglip2MultiheadAttentionPoolingHead(nn.Module): return hidden_state[:, 0] -@add_start_docstrings( - """The vision model from Siglip2 without any head or projection on top.""", - SIGLIP2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The vision model from Siglip2 without any head or projection on top. + """ ) class Siglip2VisionModel(Siglip2PreTrainedModel): config_class = Siglip2VisionConfig @@ -1032,8 +905,7 @@ class Siglip2VisionModel(Siglip2PreTrainedModel): return self.vision_model.embeddings.patch_embedding @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP2_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Siglip2VisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1043,7 +915,10 @@ class Siglip2VisionModel(Siglip2PreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: r""" - Returns: + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. Examples: @@ -1073,7 +948,7 @@ class Siglip2VisionModel(Siglip2PreTrainedModel): ) -@add_start_docstrings(SIGLIP2_START_DOCSTRING) +@auto_docstring class Siglip2Model(Siglip2PreTrainedModel): config_class = Siglip2Config @@ -1109,7 +984,7 @@ class Siglip2Model(Siglip2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SIGLIP2_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1155,7 +1030,7 @@ class Siglip2Model(Siglip2PreTrainedModel): return pooled_output - @add_start_docstrings_to_model_forward(SIGLIP2_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1165,6 +1040,11 @@ class Siglip2Model(Siglip2PreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> torch.FloatTensor: r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. + Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Siglip2VisionModel`]. @@ -1187,7 +1067,8 @@ class Siglip2Model(Siglip2PreTrainedModel): >>> with torch.no_grad(): ... image_features = model.get_image_features(**inputs) - ```""" + ``` + """ # Use Siglip2Model's config for some fields (if specified) instead of those of vision & text components. output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1207,8 +1088,7 @@ class Siglip2Model(Siglip2PreTrainedModel): return pooled_output @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Siglip2Output, config_class=Siglip2Config) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1222,7 +1102,12 @@ class Siglip2Model(Siglip2PreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> Siglip2Output: r""" - Returns: + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: @@ -1249,7 +1134,8 @@ class Siglip2Model(Siglip2PreTrainedModel): >>> probs = torch.sigmoid(logits_per_image) # these are the probabilities >>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'") 31.9% that image 0 is 'a photo of 2 cats' - ```""" + ``` + """ # Use Siglip2 model's config for some fields (if specified) instead of those of vision & text components. output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1307,12 +1193,11 @@ class Siglip2Model(Siglip2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Siglip2 vision encoder with an image classification head on top (a linear layer on top of the pooled final hidden states of the patch tokens) e.g. for ImageNet. - """, - SIGLIP2_START_DOCSTRING, + """ ) class Siglip2ForImageClassification(Siglip2PreTrainedModel): main_input_name = "pixel_values" @@ -1336,8 +1221,7 @@ class Siglip2ForImageClassification(Siglip2PreTrainedModel): self.post_init() @can_return_tuple - @add_start_docstrings_to_model_forward(SIGLIP2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -1348,13 +1232,15 @@ class Siglip2ForImageClassification(Siglip2PreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> ImageClassifierOutput: r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python @@ -1379,7 +1265,8 @@ class Siglip2ForImageClassification(Siglip2PreTrainedModel): >>> predicted_class_idx = logits.argmax(-1).item() >>> print("Predicted class:", model.config.id2label[predicted_class_idx]) Predicted class: LABEL_1 - ```""" + ``` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/siglip2/modular_siglip2.py b/src/transformers/models/siglip2/modular_siglip2.py index 23df3b0413..f9c46122b5 100644 --- a/src/transformers/models/siglip2/modular_siglip2.py +++ b/src/transformers/models/siglip2/modular_siglip2.py @@ -245,8 +245,8 @@ class Siglip2VisionTransformer(SiglipVisionTransformer): output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: r""" - Returns: - + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -323,6 +323,31 @@ class Siglip2VisionModel(SiglipVisionModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> BaseModelOutputWithPooling: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Siglip2VisionModel + + >>> model = Siglip2VisionModel.from_pretrained("google/siglip2-base-patch16-224") + >>> processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> last_hidden_state = outputs.last_hidden_state + >>> pooled_output = outputs.pooler_output # pooled features + ```""" return self.vision_model( pixel_values=pixel_values, attention_mask=pixel_attention_mask, @@ -342,6 +367,36 @@ class Siglip2Model(SiglipModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> torch.FloatTensor: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. + + Returns: + image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by + applying the projection layer to the pooled output of [`Siglip2VisionModel`]. + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, AutoModel + >>> import torch + + >>> model = AutoModel.from_pretrained("google/siglip2-base-patch16-224") + >>> processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> with torch.no_grad(): + ... image_features = model.get_image_features(**inputs) + ``` + """ # Use Siglip2Model's config for some fields (if specified) instead of those of vision & text components. output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -373,6 +428,41 @@ class Siglip2Model(SiglipModel): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> Siglip2Output: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, AutoModel + >>> import torch + + >>> model = AutoModel.from_pretrained("google/siglip2-base-patch16-224") + >>> processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> texts = ["a photo of 2 cats", "a photo of 2 dogs"] + >>> # important: we pass `padding=max_length` since the model was trained with this + >>> inputs = processor(text=texts, images=image, padding="max_length", return_tensors="pt") + + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> logits_per_image = outputs.logits_per_image + >>> probs = torch.sigmoid(logits_per_image) # these are the probabilities + >>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'") + 31.9% that image 0 is 'a photo of 2 cats' + ``` + """ # Use Siglip2 model's config for some fields (if specified) instead of those of vision & text components. output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -441,6 +531,42 @@ class Siglip2ForImageClassification(SiglipForImageClassification): output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, ) -> ImageClassifierOutput: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): + Tensor containing the spatial dimensions (height, width) of the input images. + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the image classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Examples: + + ```python + >>> from transformers import AutoImageProcessor, Siglip2ForImageClassification + >>> import torch + >>> from PIL import Image + >>> import requests + + >>> torch.manual_seed(3) # doctest: +IGNORE_RESULT + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> # note: we are loading a `Siglip2Model` from the hub here, + >>> # so the head will be randomly initialized, hence the predictions will be random if seed is not set above. + >>> image_processor = AutoImageProcessor.from_pretrained("google/siglip2-base-patch16-224") + >>> model = Siglip2ForImageClassification.from_pretrained("google/siglip2-base-patch16-224") + + >>> inputs = image_processor(images=image, return_tensors="pt") + >>> outputs = model(**inputs) + >>> logits = outputs.logits + >>> # model predicts one of the two classes + >>> predicted_class_idx = logits.argmax(-1).item() + >>> print("Predicted class:", model.config.id2label[predicted_class_idx]) + Predicted class: LABEL_1 + ``` + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/smolvlm/modeling_smolvlm.py b/src/transformers/models/smolvlm/modeling_smolvlm.py index e5abf36a3e..0f8fcc025d 100644 --- a/src/transformers/models/smolvlm/modeling_smolvlm.py +++ b/src/transformers/models/smolvlm/modeling_smolvlm.py @@ -33,44 +33,15 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, logging from ..auto import AutoModel from .configuration_smolvlm import SmolVLMConfig, SmolVLMVisionConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "SmolVLMConfig" - -SMOLVLM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SmolVLMConfig`] or [`SmolVLMVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare SmolVLM Model outputting raw hidden-states without any specific head on top.", - SMOLVLM_START_DOCSTRING, -) +@auto_docstring class SmolVLMPreTrainedModel(PreTrainedModel): config_class = SmolVLMConfig base_model_prefix = "model" @@ -404,26 +375,10 @@ class SmolVLMEncoder(nn.Module): ) -SMOLVLM_VISION_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SmolVLMVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The SmolVLM Vision Transformer Model outputting raw image embedding.", - SMOLVLM_VISION_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The SmolVLM Vision Transformer Model outputting raw image embedding. + """ ) class SmolVLMVisionTransformer(SmolVLMPreTrainedModel): config_class = SmolVLMVisionConfig @@ -578,9 +533,10 @@ class SmolVLMConnector(nn.Module): return image_hidden_states -@add_start_docstrings( - """SmolVLM model consisting of a SIGLIP vision encoder and Llama3 language decoder""", - SMOLVLM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SmolVLM model consisting of a SIGLIP vision encoder and Llama3 language decoder + """ ) class SmolVLMModel(SmolVLMPreTrainedModel): """ @@ -677,6 +633,17 @@ class SmolVLMModel(SmolVLMPreTrainedModel): return merged_embeds @can_return_tuple + @auto_docstring( + custom_intro=""" + Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to + the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where + max_num_images is the maximum number of images among the batch_size samples in the batch. + Padding images are not needed beyond padding the pixel_values at the entrance of the model. + For efficiency, we only pass through the vision_model's forward the real images by + discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where + image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3. + """ + ) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -694,6 +661,12 @@ class SmolVLMModel(SmolVLMPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, SmolVLMBaseModelOutputWithPast]: + r""" + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The hidden states of the image encoder after modality projection. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -848,90 +821,12 @@ class SmolVLMCausalLMOutputWithPast(ModelOutput): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -SMOLVLM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`CLIPImageProcessor`] for processing images). - pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): - Mask to avoid performing attention on padding pixel indices. - image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): - The hidden states of the image encoder after modality projection. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The SmolVLM Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. """, - SMOLVLM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The SmolVLM Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. + """ ) class SmolVLMForConditionalGeneration(SmolVLMPreTrainedModel, GenerationMixin): - """ - A subclass of Idefics3ForConditionalGeneration that uses SmolVLMModel - instead of the default Idefics3Model. - """ - _tied_weights_keys = ["lm_head.weight"] def __init__(self, config): @@ -975,8 +870,7 @@ class SmolVLMForConditionalGeneration(SmolVLMPreTrainedModel, GenerationMixin): self.lm_head = new_embeddings @can_return_tuple - @add_start_docstrings_to_model_forward(SMOLVLM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SmolVLMCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -997,13 +891,15 @@ class SmolVLMForConditionalGeneration(SmolVLMPreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, SmolVLMCausalLMOutputWithPast]: r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `SmolVLMForConditionalGeneration`). - Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The hidden states of the image encoder after modality projection. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `SmolVLMForConditionalGeneration`). + Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only + computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 66f7775261..95d3c4dada 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -22,10 +22,7 @@ from torch import nn from ...cache_utils import DynamicCache from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...processing_utils import Unpack -from ...utils import ( - can_return_tuple, - logging, -) +from ...utils import logging from ..idefics3.configuration_idefics3 import Idefics3Config, Idefics3VisionConfig from ..idefics3.image_processing_idefics3 import Idefics3ImageProcessor from ..idefics3.modeling_idefics3 import ( @@ -198,7 +195,6 @@ class SmolVLMModel(Idefics3Model): merged_embeds = torch.where(image_mask.unsqueeze(-1), image_embeds, inputs_embeds) return merged_embeds - @can_return_tuple def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -330,11 +326,6 @@ class SmolVLMModel(Idefics3Model): class SmolVLMForConditionalGeneration(Idefics3ForConditionalGeneration): - """ - A subclass of Idefics3ForConditionalGeneration that uses SmolVLMModel - instead of the default Idefics3Model. - """ - def __init__(self, config): super().__init__(config) self.model = SmolVLMModel(config) @@ -343,14 +334,6 @@ class SmolVLMForConditionalGeneration(Idefics3ForConditionalGeneration): def forward(self, **super_kwargs): r""" - Args: - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `SmolVLMForConditionalGeneration`). - Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only - computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: - Example: ```python diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py index 9cab18104e..db131a2cbf 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py @@ -24,7 +24,7 @@ from ...configuration_utils import PretrainedConfig from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutput, Seq2SeqLMOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel, AutoModelForCausalLM from .configuration_speech_encoder_decoder import SpeechEncoderDecoderConfig @@ -32,123 +32,6 @@ from .configuration_speech_encoder_decoder import SpeechEncoderDecoderConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "SpeechEncoderDecoderConfig" - -SPEECH_ENCODER_DECODER_START_DOCSTRING = r""" - This class can be used to initialize a speech-sequence-to-text-sequence model with any pretrained speech - autoencoding model as the encoder and any pretrained text autoregressive model as the decoder. The encoder is - loaded via [`~AutoModel.from_pretrained`] function and the decoder is loaded via - [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added to the decoder - and should be fine-tuned on a downstream generative task, like summarization. - - The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation - tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation - Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi - Zhou, Wei Li, Peter J. Liu. - - Additionally, in [Large-Scale Self- and Semi-Supervised Learning for Speech - Translation](https://arxiv.org/abs/2104.06678) it is shown how leveraging large pretrained speech models for speech - translation yields a significant performance improvement. - - After such an Speech-Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other - models (see the examples for more information). - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SpeechEncoderDecoderConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING = r""" - Args: - inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, feature_dim)`, *optional*): - Float values of input raw speech waveform or speech features. Values can be obtained by loading a `.flac` - or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile - library (`pip install soundfile`). To prepare the array into `inputs`, either the [`Wav2Vec2Processor`] or - [`Speech2TextProcessor`] should be used for padding and conversion into a tensor of type - `torch.FloatTensor`. - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the - right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`. - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - encoder_outputs (`tuple(torch.FloatTensor)`, *optional*): - This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor - of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the - decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. This is useful if you want more control over how to convert `decoder_input_ids` indices - into associated vectors than the model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, - ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file - into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install - soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding - and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*): - Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained - by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* - via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`Speech2TextFeatureExtractor`] should be used for extracting the fbank features, padding and conversion - into a tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] - return_dict (`bool`, *optional*): - If set to `True`, the model will return a [`~utils.Seq2SeqLMOutput`] instead of a plain tuple. - kwargs (*optional*): Remaining dictionary of keyword arguments. Keyword arguments come in two flavors: - - - Without a prefix which will be input as `**encoder_kwargs` for the encoder forward function. - - With a *decoder_* prefix which will be input as `**decoder_kwargs` for the decoder forward function. -""" - # Copied from transformers.models.encoder_decoder.modeling_encoder_decoder.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -169,7 +52,7 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start return shifted_input_ids -@add_start_docstrings(SPEECH_ENCODER_DECODER_START_DOCSTRING) +@auto_docstring class SpeechEncoderDecoderModel(PreTrainedModel, GenerationMixin): r""" [`SpeechEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with @@ -192,6 +75,12 @@ class SpeechEncoderDecoderModel(PreTrainedModel, GenerationMixin): encoder: Optional[PreTrainedModel] = None, decoder: Optional[PreTrainedModel] = None, ): + r""" + encoder (`PreTrainedModel`, *optional*): + The encoder model to use. + decoder (`PreTrainedModel`, *optional*): + The decoder model to use. + """ if config is None and (encoder is None or decoder is None): raise ValueError("Either a configuration or an encoder and a decoder has to be provided.") if config is None: @@ -428,8 +317,7 @@ class SpeechEncoderDecoderModel(PreTrainedModel, GenerationMixin): config.tie_word_embeddings = False return cls(encoder=encoder, decoder=decoder, config=config) - @add_start_docstrings_to_model_forward(SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, inputs: Optional[torch.FloatTensor] = None, @@ -449,7 +337,47 @@ class SpeechEncoderDecoderModel(PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" - Returns: + inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, feature_dim)`, *optional*): + Float values of input raw speech waveform or speech features. Values can be obtained by loading a `.flac` + or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile + library (`pip install soundfile`). To prepare the array into `inputs`, either the [`Wav2Vec2Processor`] or + [`Speech2TextProcessor`] should be used for padding and conversion into a tensor of type + `torch.FloatTensor`. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the + right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`. + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert `decoder_input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, + ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file + into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install + soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding + and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details. + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*): + Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained + by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* + via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`Speech2TextFeatureExtractor`] should be used for extracting the fbank features, padding and conversion + into a tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] Examples: diff --git a/src/transformers/models/speech_to_text/modeling_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_speech_to_text.py index 397c4f3886..b5842217fc 100755 --- a/src/transformers/models/speech_to_text/modeling_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_speech_to_text.py @@ -32,18 +32,14 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_speech_to_text import Speech2TextConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Speech2TextConfig" - # Copied from transformers.models.bart.modeling_bart.shift_tokens_right def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): @@ -517,6 +513,7 @@ class Speech2TextDecoderLayer(nn.Module): return outputs +@auto_docstring class Speech2TextPreTrainedModel(PreTrainedModel): config_class = Speech2TextConfig base_model_prefix = "model" @@ -562,108 +559,6 @@ class Speech2TextPreTrainedModel(PreTrainedModel): return attention_mask -SPEECH_TO_TEXT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Speech2TextConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SPEECH_TO_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): - Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained - by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* - via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`SpeechToTextTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - SpeechToText uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read - [`modeling_speech_to_text._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the - paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class Speech2TextEncoder(Speech2TextPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1069,10 +964,7 @@ class Speech2TextDecoder(Speech2TextPreTrainedModel): ) -@add_start_docstrings( - "The bare Speech2Text Model outputting raw hidden-states without any specific head on top.", - SPEECH_TO_TEXT_START_DOCSTRING, -) +@auto_docstring class Speech2TextModel(Speech2TextPreTrainedModel): def __init__(self, config: Speech2TextConfig): super().__init__(config) @@ -1095,8 +987,7 @@ class Speech2TextModel(Speech2TextPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(SPEECH_TO_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_features: Optional[torch.LongTensor] = None, @@ -1115,7 +1006,35 @@ class Speech2TextModel(Speech2TextPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): + Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained + by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* + via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`SpeechToTextTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + SpeechToText uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_speech_to_text._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1200,9 +1119,10 @@ class Speech2TextModel(Speech2TextPreTrainedModel): ) -@add_start_docstrings( - "The Speech2Text Model with a language modeling head. Can be used for summarization.", - SPEECH_TO_TEXT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Speech2Text Model with a language modeling head. Can be used for summarization. + """ ) class Speech2TextForConditionalGeneration(Speech2TextPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -1228,8 +1148,7 @@ class Speech2TextForConditionalGeneration(Speech2TextPreTrainedModel, Generation def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(SPEECH_TO_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_features: Optional[torch.LongTensor] = None, @@ -1249,13 +1168,40 @@ class Speech2TextForConditionalGeneration(Speech2TextPreTrainedModel, Generation return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): + Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained + by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* + via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~Speech2TextFeatureExtractor.__call__`] + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`SpeechToTextTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + SpeechToText uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_speech_to_text._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - Returns: - Example: ```python diff --git a/src/transformers/models/speecht5/modeling_speecht5.py b/src/transformers/models/speecht5/modeling_speecht5.py index 8d784487ab..5004dd037c 100644 --- a/src/transformers/models/speecht5/modeling_speecht5.py +++ b/src/transformers/models/speecht5/modeling_speecht5.py @@ -36,7 +36,7 @@ from ...modeling_outputs import ( Seq2SeqSpectrogramOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_speecht5 import SpeechT5Config, SpeechT5HifiGanConfig @@ -46,7 +46,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 1 # General docstring -_CONFIG_FOR_DOC = "SpeechT5Config" # Copied from transformers.models.bart.modeling_bart.shift_tokens_right @@ -1202,12 +1201,8 @@ class SpeechT5DecoderLayer(nn.Module): return outputs +@auto_docstring class SpeechT5PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SpeechT5Config base_model_prefix = "speecht5" main_input_name = "input_values" @@ -1962,136 +1957,10 @@ class SpeechT5SpectrogramLoss(nn.Module): return loss -SPEECHT5_BASE_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SpeechT5Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. - encoder ([`SpeechT5EncoderWithSpeechPrenet`] or [`SpeechT5EncoderWithTextPrenet`] or `None`): - The Transformer encoder module that applies the appropriate speech or text encoder prenet. If `None`, - [`SpeechT5EncoderWithoutPrenet`] will be used and the `input_values` are assumed to be hidden states. - decoder ([`SpeechT5DecoderWithSpeechPrenet`] or [`SpeechT5DecoderWithTextPrenet`] or `None`): - The Transformer decoder module that applies the appropriate speech or text decoder prenet. If `None`, - [`SpeechT5DecoderWithoutPrenet`] will be used and the `decoder_input_values` are assumed to be hidden - states. -""" - - -SPEECHT5_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SpeechT5Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -SPEECHT5_INPUTS_DOCSTRING = r""" - Args: - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_values`. Causal mask will - also be used by default. - - If you want to change padding behavior, you should read [`SpeechT5Decoder._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - head_mask (`torch.FloatTensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_values` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_values` of shape `(batch_size, sequence_length)`. decoder_inputs_embeds (`torch.FloatTensor` - of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing - `decoder_input_values` you can choose to directly pass an embedded representation. If `past_key_values` is - used, optionally only the last `decoder_inputs_embeds` have to be input (see `past_key_values`). This is - useful if you want more control over how to convert `decoder_input_values` indices into associated vectors - than the model's internal embedding lookup matrix. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SpeechT5 Encoder-Decoder Model outputting raw hidden-states without any specific pre- or post-nets.", - SPEECHT5_BASE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare SpeechT5 Encoder-Decoder Model outputting raw hidden-states without any specific pre- or post-nets. + """ ) class SpeechT5Model(SpeechT5PreTrainedModel): def __init__( @@ -2100,6 +1969,12 @@ class SpeechT5Model(SpeechT5PreTrainedModel): encoder: Optional[nn.Module] = None, decoder: Optional[nn.Module] = None, ): + r""" + encoder (`PreTrainedModel`, *optional*): + The encoder model to use. + decoder (`PreTrainedModel`, *optional*): + The decoder model to use. + """ super().__init__(config) self.config = config self.encoder = SpeechT5EncoderWithoutPrenet(config) if encoder is None else encoder @@ -2135,8 +2010,7 @@ class SpeechT5Model(SpeechT5PreTrainedModel): if isinstance(self.encoder, SpeechT5EncoderWithSpeechPrenet): self.encoder.prenet.freeze_feature_encoder() - @add_start_docstrings_to_model_forward(SPEECHT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor] = None, @@ -2158,16 +2032,24 @@ class SpeechT5Model(SpeechT5PreTrainedModel): input_values (`torch.Tensor` of shape `(batch_size, sequence_length)`): Depending on which encoder is being used, the `input_values` are either: float values of the input raw speech waveform, or indices of input sequence tokens in the vocabulary, or hidden states. - decoder_input_values (`torch.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Depending on which decoder is being used, the `decoder_input_values` are either: float values of log-mel filterbank features extracted from the raw speech waveform, or indices of decoder input sequence tokens in the vocabulary, or hidden states. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_values`. Causal mask will + also be used by default. + If you want to change padding behavior, you should read [`SpeechT5Decoder._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. - - Returns: """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -2237,9 +2119,10 @@ class SpeechT5Model(SpeechT5PreTrainedModel): ) -@add_start_docstrings( - """SpeechT5 Model with a speech encoder and a text decoder.""", - SPEECHT5_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SpeechT5 Model with a speech encoder and a text decoder. + """ ) class SpeechT5ForSpeechToText(SpeechT5PreTrainedModel, GenerationMixin): _tied_weights_keys = ["text_decoder_postnet.lm_head.weight"] @@ -2283,8 +2166,7 @@ class SpeechT5ForSpeechToText(SpeechT5PreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.text_decoder_postnet.set_output_embeddings(new_embeddings) - @add_start_docstrings_to_model_forward(SPEECHT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.FloatTensor] = None, @@ -2308,7 +2190,6 @@ class SpeechT5ForSpeechToText(SpeechT5PreTrainedModel, GenerationMixin): into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (*pip install soundfile*). To prepare the array into `input_values`, the [`SpeechT5Processor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`SpeechT5Processor.__call__`] for details. - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. @@ -2320,7 +2201,18 @@ class SpeechT5ForSpeechToText(SpeechT5PreTrainedModel, GenerationMixin): SpeechT5 uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_values`. Causal mask will + also be used by default. + If you want to change padding behavior, you should read [`SpeechT5Decoder._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is @@ -2329,8 +2221,6 @@ class SpeechT5ForSpeechToText(SpeechT5PreTrainedModel, GenerationMixin): Label indices can be obtained using [`SpeechT5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - Returns: - Example: ```python @@ -2565,9 +2455,10 @@ def _generate_speech( return outputs -@add_start_docstrings( - """SpeechT5 Model with a text encoder and a speech decoder.""", - SPEECHT5_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SpeechT5 Model with a text encoder and a speech decoder. + """ ) class SpeechT5ForTextToSpeech(SpeechT5PreTrainedModel): main_input_name = "input_ids" @@ -2605,8 +2496,7 @@ class SpeechT5ForTextToSpeech(SpeechT5PreTrainedModel): def get_decoder(self): return self.speecht5.get_decoder() - @add_start_docstrings_to_model_forward(SPEECHT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqSpectrogramOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2640,14 +2530,26 @@ class SpeechT5ForTextToSpeech(SpeechT5PreTrainedModel): SpeechT5 uses an all-zero spectrum as the starting token for `decoder_input_values` generation. If `past_key_values` is used, optionally only the last `decoder_input_values` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_values`. Causal mask will + also be used by default. + + If you want to change padding behavior, you should read [`SpeechT5Decoder._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. labels (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_mel_bins)`, *optional*): Float values of target mel spectrogram. Timesteps set to `-100.0` are ignored (masked) for the loss computation. Spectrograms can be obtained using [`SpeechT5Processor`]. See [`SpeechT5Processor.__call__`] for details. - - Returns: + stop_labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Binary tensor indicating the position of the stop token in the sequence. Example: @@ -2916,9 +2818,10 @@ class SpeechT5ForTextToSpeech(SpeechT5PreTrainedModel): ) -@add_start_docstrings( - """SpeechT5 Model with a speech encoder and a speech decoder.""", - SPEECHT5_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + SpeechT5 Model with a speech encoder and a speech decoder. + """ ) class SpeechT5ForSpeechToSpeech(SpeechT5PreTrainedModel): def __init__(self, config: SpeechT5Config): @@ -2946,8 +2849,7 @@ class SpeechT5ForSpeechToSpeech(SpeechT5PreTrainedModel): """ self.get_encoder().prenet.freeze_feature_encoder() - @add_start_docstrings_to_model_forward(SPEECHT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqSpectrogramOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.FloatTensor] = None, @@ -2979,13 +2881,25 @@ class SpeechT5ForSpeechToSpeech(SpeechT5PreTrainedModel): SpeechT5 uses an all-zero spectrum as the starting token for `decoder_input_values` generation. If `past_key_values` is used, optionally only the last `decoder_input_values` have to be input (see `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_values`. Causal mask will + also be used by default. + + If you want to change padding behavior, you should read [`SpeechT5Decoder._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. labels (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_mel_bins)`, *optional*): Float values of target mel spectrogram. Spectrograms can be obtained using [`SpeechT5Processor`]. See [`SpeechT5Processor.__call__`] for details. - - Returns: + stop_labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Binary tensor indicating the position of the stop token in the sequence. Example: @@ -3152,23 +3066,6 @@ class SpeechT5ForSpeechToSpeech(SpeechT5PreTrainedModel): ) -HIFIGAN_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SpeechT5HifiGanConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - class HifiGanResidualBlock(nn.Module): def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5), leaky_relu_slope=0.1): super().__init__() @@ -3231,9 +3128,10 @@ class HifiGanResidualBlock(nn.Module): return hidden_states -@add_start_docstrings( - """HiFi-GAN vocoder.""", - HIFIGAN_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + HiFi-GAN vocoder. + """ ) class SpeechT5HifiGan(PreTrainedModel): config_class = SpeechT5HifiGanConfig @@ -3304,16 +3202,18 @@ class SpeechT5HifiGan(PreTrainedModel): layer.remove_weight_norm() nn.utils.remove_weight_norm(self.conv_post) - def forward(self, spectrogram: torch.FloatTensor) -> torch.FloatTensor: - r""" + @auto_docstring( + custom_intro=""" Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech waveform. - - Args: - spectrogram (`torch.FloatTensor`): - Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, - config.model_in_dim)`, or un-batched and of shape `(sequence_length, config.model_in_dim)`. + """ + ) + def forward(self, spectrogram: torch.FloatTensor) -> torch.FloatTensor: + r""" + spectrogram (`torch.FloatTensor`): + Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, + config.model_in_dim)`, or un-batched and of shape `(sequence_length, config.model_in_dim)`. Returns: `torch.FloatTensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of diff --git a/src/transformers/models/splinter/modeling_splinter.py b/src/transformers/models/splinter/modeling_splinter.py index 174e766598..6498947325 100755 --- a/src/transformers/models/splinter/modeling_splinter.py +++ b/src/transformers/models/splinter/modeling_splinter.py @@ -27,15 +27,15 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, ModelOutput, QuestionAnsweringModelOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import ( + auto_docstring, + logging, +) from .configuration_splinter import SplinterConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "tau/splinter-base" -_CONFIG_FOR_DOC = "SplinterConfig" - class SplinterEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings.""" @@ -510,12 +510,8 @@ class SplinterEncoder(nn.Module): ) +@auto_docstring class SplinterPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SplinterConfig base_model_prefix = "splinter" supports_gradient_checkpointing = True @@ -537,71 +533,7 @@ class SplinterPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -SPLINTER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SplinterConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SPLINTER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `{0}`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `{0}`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `{0}`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Splinter Model transformer outputting raw hidden-states without any specific head on top.", - SPLINTER_START_DOCSTRING, -) +@auto_docstring class SplinterModel(SplinterPreTrainedModel): """ The model is an encoder (with only self-attention) following the architecture described in [Attention is all you @@ -633,12 +565,7 @@ class SplinterModel(SplinterPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SPLINTER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -656,23 +583,19 @@ class SplinterModel(SplinterPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -817,13 +740,7 @@ class QuestionAwareSpanSelectionHead(nn.Module): return start_logits, end_logits -@add_start_docstrings( - """ - Splinter Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - SPLINTER_START_DOCSTRING, -) +@auto_docstring class SplinterForQuestionAnswering(SplinterPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -835,12 +752,7 @@ class SplinterForQuestionAnswering(SplinterPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SPLINTER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -857,14 +769,19 @@ class SplinterForQuestionAnswering(SplinterPreTrainedModel): question_positions: Optional[torch.LongTensor] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be @@ -970,13 +887,12 @@ class SplinterForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Splinter Model for the recurring span selection task as done during the pretraining. The difference to the QA task is that we do not have a question, but multiple question tokens that replace the occurrences of recurring spans instead. - """, - SPLINTER_START_DOCSTRING, + """ ) class SplinterForPreTraining(SplinterPreTrainedModel): def __init__(self, config): @@ -989,9 +905,7 @@ class SplinterForPreTraining(SplinterPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - SPLINTER_INPUTS_DOCSTRING.format("batch_size, num_questions, sequence_length") - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1008,6 +922,30 @@ class SplinterForPreTraining(SplinterPreTrainedModel): question_positions: Optional[torch.LongTensor] = None, ) -> Union[Tuple, SplinterForPreTrainingOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_questions, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. start_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence diff --git a/src/transformers/models/squeezebert/modeling_squeezebert.py b/src/transformers/models/squeezebert/modeling_squeezebert.py index efb29035ba..9b71d118f4 100644 --- a/src/transformers/models/squeezebert/modeling_squeezebert.py +++ b/src/transformers/models/squeezebert/modeling_squeezebert.py @@ -32,15 +32,15 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import ( + auto_docstring, + logging, +) from .configuration_squeezebert import SqueezeBertConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "squeezebert/squeezebert-uncased" -_CONFIG_FOR_DOC = "SqueezeBertConfig" - class SqueezeBertEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings.""" @@ -415,12 +415,8 @@ class SqueezeBertOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class SqueezeBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SqueezeBertConfig base_model_prefix = "transformer" @@ -443,105 +439,7 @@ class SqueezeBertPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -SQUEEZEBERT_START_DOCSTRING = r""" - - The SqueezeBERT model was proposed in [SqueezeBERT: What can computer vision teach NLP about efficient neural - networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. - Keutzer - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - For best results finetuning SqueezeBERT on text classification tasks, it is recommended to use the - *squeezebert/squeezebert-mnli-headless* checkpoint as a starting point. - - Parameters: - config ([`SqueezeBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. - - Hierarchy: - - ``` - Internal class hierarchy: - SqueezeBertModel - SqueezeBertEncoder - SqueezeBertModule - SqueezeBertSelfAttention - ConvActivation - ConvDropoutLayerNorm - ``` - - Data layouts: - - ``` - Input data is in [batch, sequence_length, hidden_size] format. - - Data inside the encoder is in [batch, hidden_size, sequence_length] format. But, if `output_hidden_states == True`, the data from inside the encoder is returned in [batch, sequence_length, hidden_size] format. - - The final output of the encoder is in [batch, sequence_length, hidden_size] format. - ``` -""" - -SQUEEZEBERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SqueezeBERT Model transformer outputting raw hidden-states without any specific head on top.", - SQUEEZEBERT_START_DOCSTRING, -) +@auto_docstring class SqueezeBertModel(SqueezeBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -567,12 +465,7 @@ class SqueezeBertModel(SqueezeBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -641,7 +534,7 @@ class SqueezeBertModel(SqueezeBertPreTrainedModel): ) -@add_start_docstrings("""SqueezeBERT Model with a `language modeling` head on top.""", SQUEEZEBERT_START_DOCSTRING) +@auto_docstring class SqueezeBertForMaskedLM(SqueezeBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -661,12 +554,7 @@ class SqueezeBertForMaskedLM(SqueezeBertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -720,12 +608,11 @@ class SqueezeBertForMaskedLM(SqueezeBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" SqueezeBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - SQUEEZEBERT_START_DOCSTRING, + """ ) class SqueezeBertForSequenceClassification(SqueezeBertPreTrainedModel): def __init__(self, config): @@ -740,12 +627,7 @@ class SqueezeBertForSequenceClassification(SqueezeBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -819,13 +701,7 @@ class SqueezeBertForSequenceClassification(SqueezeBertPreTrainedModel): ) -@add_start_docstrings( - """ - SqueezeBERT Model with a multiple choice classification head on top (a linear layer on top of the pooled output and - a softmax) e.g. for RocStories/SWAG tasks. - """, - SQUEEZEBERT_START_DOCSTRING, -) +@auto_docstring class SqueezeBertForMultipleChoice(SqueezeBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -837,14 +713,7 @@ class SqueezeBertForMultipleChoice(SqueezeBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -859,6 +728,30 @@ class SqueezeBertForMultipleChoice(SqueezeBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see @@ -912,13 +805,7 @@ class SqueezeBertForMultipleChoice(SqueezeBertPreTrainedModel): ) -@add_start_docstrings( - """ - SqueezeBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - SQUEEZEBERT_START_DOCSTRING, -) +@auto_docstring class SqueezeBertForTokenClassification(SqueezeBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -931,12 +818,7 @@ class SqueezeBertForTokenClassification(SqueezeBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -990,13 +872,7 @@ class SqueezeBertForTokenClassification(SqueezeBertPreTrainedModel): ) -@add_start_docstrings( - """ - SqueezeBERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - SQUEEZEBERT_START_DOCSTRING, -) +@auto_docstring class SqueezeBertForQuestionAnswering(SqueezeBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1008,12 +884,7 @@ class SqueezeBertForQuestionAnswering(SqueezeBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SQUEEZEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1028,16 +899,6 @@ class SqueezeBertForQuestionAnswering(SqueezeBertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.transformer( diff --git a/src/transformers/models/stablelm/modeling_stablelm.py b/src/transformers/models/stablelm/modeling_stablelm.py index 7350cc1cc8..001e0b8dea 100755 --- a/src/transformers/models/stablelm/modeling_stablelm.py +++ b/src/transformers/models/stablelm/modeling_stablelm.py @@ -39,15 +39,7 @@ from ...modeling_outputs import ( ) from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_stablelm import StableLmConfig @@ -63,9 +55,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "stabilityai/stablelm-3b-4e1t" -_CONFIG_FOR_DOC = "StableLmConfig" - # Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->StableLm class StableLmRotaryEmbedding(nn.Module): @@ -622,27 +611,7 @@ class StableLmDecoderLayer(nn.Module): return outputs -STABLELM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`StableLmConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare StableLm Model outputting raw hidden-states without any specific head on top.", - STABLELM_START_DOCSTRING, -) +@auto_docstring class StableLmPreTrainedModel(PreTrainedModel): config_class = StableLmConfig base_model_prefix = "model" @@ -670,85 +639,7 @@ class StableLmPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -STABLELM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - Two formats are allowed: - - a [`~cache_utils.Cache`] instance, see our - [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy - cache format. - - The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the - legacy cache format will be returned. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare StableLm Model outputting raw hidden-states without any specific head on top.", - STABLELM_START_DOCSTRING, -) +@auto_docstring class StableLmModel(StableLmPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`StableLmDecoderLayer`] @@ -781,7 +672,7 @@ class StableLmModel(StableLmPreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(STABLELM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1067,8 +958,7 @@ class StableLmForCausalLM(StableLmPreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(STABLELM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring # Ignore copy def forward( self, @@ -1086,35 +976,26 @@ class StableLmForCausalLM(StableLmPreTrainedModel, GenerationMixin): **kwargs, ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, StableLmForCausalLM - >>> model = StableLmForCausalLM.from_pretrained("stabilityai/stablelm-3b-4e1t") - >>> tokenizer = AutoTokenizer.from_pretrained("stabilityai/stablelm-3b-4e1t") + >>> model = StableLmForCausalLM.from_pretrained("adept/persimmon-8b-base") + >>> tokenizer = AutoTokenizer.from_pretrained("adept/persimmon-8b-base") - >>> prompt = "The weather is always wonderful in" + >>> prompt = "human: Hey, what should I eat for dinner?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - 'The weather is always wonderful in the summer in the city of San Diego. The city is located on the coast of the Pacific Ocean and is surrounded by' + 'human: Hey, what should I eat for dinner?\n\ncat: 🐱\n\nhuman: 😐\n\n' ```""" output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -1122,6 +1003,7 @@ class StableLmForCausalLM(StableLmPreTrainedModel, GenerationMixin): output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) outputs: BaseModelOutputWithPast = self.model( input_ids=input_ids, attention_mask=attention_mask, @@ -1157,8 +1039,8 @@ class StableLmForCausalLM(StableLmPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The StableLm transformer with a sequence classification head on top (linear layer). [`StableLmForSequenceClassification`] uses the last token in order to do the classification, as other causal @@ -1169,8 +1051,7 @@ class StableLmForCausalLM(StableLmPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - STABLELM_START_DOCSTRING, + """ ) # Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with LLAMA->STABLELM,Llama->StableLm class StableLmForSequenceClassification(StableLmPreTrainedModel): @@ -1190,7 +1071,7 @@ class StableLmForSequenceClassification(StableLmPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(STABLELM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1259,13 +1140,7 @@ class StableLmForSequenceClassification(StableLmPreTrainedModel): ) -@add_start_docstrings( - """ - The StableLm Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - STABLELM_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.llama.modeling_llama.LlamaForTokenClassification with Llama->StableLm, LLAMA->STABLELM class StableLmForTokenClassification(StableLmPreTrainedModel): def __init__(self, config): @@ -1291,12 +1166,7 @@ class StableLmForTokenClassification(StableLmPreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(STABLELM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/starcoder2/modeling_starcoder2.py b/src/transformers/models/starcoder2/modeling_starcoder2.py index b56a4fcf68..50317b78f8 100644 --- a/src/transformers/models/starcoder2/modeling_starcoder2.py +++ b/src/transformers/models/starcoder2/modeling_starcoder2.py @@ -44,16 +44,7 @@ from ...modeling_outputs import ( from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torch_flex_attn_available, logging from .configuration_starcoder2 import Starcoder2Config @@ -64,9 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "bigcode/starcoder2-7b" - -_CONFIG_FOR_DOC = "Starcoder2Config" class Starcoder2MLP(nn.Module): @@ -316,27 +304,7 @@ class Starcoder2RotaryEmbedding(nn.Module): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -STARCODER2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Starcoder2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Starcoder2 Model outputting raw hidden-states without any specific head on top.", - STARCODER2_START_DOCSTRING, -) +@auto_docstring class Starcoder2PreTrainedModel(PreTrainedModel): config_class = Starcoder2Config base_model_prefix = "model" @@ -366,88 +334,8 @@ class Starcoder2PreTrainedModel(PreTrainedModel): module.bias.data.zero_() -STARCODER2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length) or `BlockMask`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - If the model is configured to use flex_attention, it will attempt to convert the mask Tensor into a BlockMask, - but you can also pass a `BlockMask` object directly here. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Cache`, *optional*): - Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` - returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. - - It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't - have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` - of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Starcoder2 Model outputting raw hidden-states without any specific head on top.", - STARCODER2_START_DOCSTRING, -) +@auto_docstring class Starcoder2Model(Starcoder2PreTrainedModel): - """ - Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Starcoder2DecoderLayer`] - - Args: - config: Starcoder2Config - """ - def __init__(self, config: Starcoder2Config): super().__init__(config) self.padding_idx = config.pad_token_id @@ -472,7 +360,7 @@ class Starcoder2Model(Starcoder2PreTrainedModel): self.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(STARCODER2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -724,6 +612,7 @@ class Starcoder2Model(Starcoder2PreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... +@auto_docstring class Starcoder2ForCausalLM(Starcoder2PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} @@ -757,8 +646,7 @@ class Starcoder2ForCausalLM(Starcoder2PreTrainedModel, GenerationMixin): return self.model @can_return_tuple - @add_start_docstrings_to_model_forward(STARCODER2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -775,19 +663,10 @@ class Starcoder2ForCausalLM(Starcoder2PreTrainedModel, GenerationMixin): **kwargs: Unpack[KwargsForCausalLM], ) -> CausalLMOutputWithPast: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -842,8 +721,8 @@ class Starcoder2ForCausalLM(Starcoder2PreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Starcoder2 Model transformer with a sequence classification head on top (linear layer). [`Starcoder2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -854,8 +733,7 @@ class Starcoder2ForCausalLM(Starcoder2PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - STARCODER2_START_DOCSTRING, + """ ) class Starcoder2ForSequenceClassification(Starcoder2PreTrainedModel): def __init__(self, config): @@ -874,7 +752,7 @@ class Starcoder2ForSequenceClassification(Starcoder2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(STARCODER2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -943,13 +821,7 @@ class Starcoder2ForSequenceClassification(Starcoder2PreTrainedModel): ) -@add_start_docstrings( - """ - The Starcoder2 Model transformer with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - STARCODER2_START_DOCSTRING, -) +@auto_docstring class Starcoder2ForTokenClassification(Starcoder2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -974,12 +846,7 @@ class Starcoder2ForTokenClassification(Starcoder2PreTrainedModel): self.model.embed_tokens = value @can_return_tuple - @add_start_docstrings_to_model_forward(STARCODER2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/starcoder2/modular_starcoder2.py b/src/transformers/models/starcoder2/modular_starcoder2.py index 77b05d1dc1..ad1ff32875 100644 --- a/src/transformers/models/starcoder2/modular_starcoder2.py +++ b/src/transformers/models/starcoder2/modular_starcoder2.py @@ -28,12 +28,10 @@ from torch import nn from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - BaseModelOutputWithPast, -) +from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack -from ...utils import add_start_docstrings_to_model_forward, can_return_tuple, logging +from ...utils import logging from ..mistral.modeling_mistral import ( MistralAttention, MistralDecoderLayer, @@ -51,9 +49,6 @@ from .configuration_starcoder2 import Starcoder2Config logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Starcoder2Config" -_CHECKPOINT_FOR_DOC = "bigcode/starcoder2-7b" - class Starcoder2MLP(nn.Module): def __init__(self, config: Starcoder2Config): @@ -165,9 +160,6 @@ class Starcoder2PreTrainedModel(MistralPreTrainedModel): module.bias.data.zero_() -STARCODER2_INPUTS_DOCSTRING = None # will be automatically redefined - - class Starcoder2Model(MistralModel): def __init__(self, config: Starcoder2Config): super().__init__(config) @@ -177,8 +169,6 @@ class Starcoder2Model(MistralModel): self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon) self.embedding_dropout = config.embedding_dropout - @can_return_tuple - @add_start_docstrings_to_model_forward(STARCODER2_INPUTS_DOCSTRING) def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/superglue/modeling_superglue.py b/src/transformers/models/superglue/modeling_superglue.py index 049eb91b84..7b1f335f7f 100644 --- a/src/transformers/models/superglue/modeling_superglue.py +++ b/src/transformers/models/superglue/modeling_superglue.py @@ -20,19 +20,16 @@ from typing import Optional, Tuple, Union import torch from torch import nn -from transformers import PreTrainedModel, add_start_docstrings +from transformers import PreTrainedModel from transformers.models.superglue.configuration_superglue import SuperGlueConfig from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ModelOutput, add_start_docstrings_to_model_forward, logging +from ...utils import ModelOutput, auto_docstring, logging from ..auto import AutoModelForKeypointDetection logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC_ = "SuperGlueConfig" -_CHECKPOINT_FOR_DOC_ = "magic-leap-community/superglue_indoor" - def concat_pairs(tensor_tuple0: Tuple[torch.Tensor], tensor_tuple1: Tuple[torch.Tensor]) -> Tuple[torch.Tensor]: """ @@ -544,12 +541,8 @@ class SuperGlueFinalProjection(nn.Module): return self.final_proj(descriptors) +@auto_docstring class SuperGluePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SuperGlueConfig base_model_prefix = "superglue" main_input_name = "pixel_values" @@ -569,35 +562,10 @@ class SuperGluePreTrainedModel(PreTrainedModel): nn.init.constant_(module.linear.bias, 0.0) -SUPERGLUE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SuperGlueConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +@auto_docstring( + custom_intro=""" + SuperGlue model taking images as inputs and outputting the matching of them. """ - -SUPERGLUE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SuperGlueImageProcessor`]. See - [`SuperGlueImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors. See `attentions` under returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "SuperGlue model taking images as inputs and outputting the matching of them.", - SUPERGLUE_START_DOCSTRING, ) class SuperGlueForKeypointMatching(SuperGluePreTrainedModel): """SuperGlue feature matching middle-end @@ -776,7 +744,7 @@ class SuperGlueForKeypointMatching(SuperGluePreTrainedModel): all_attentions, ) - @add_start_docstrings_to_model_forward(SUPERGLUE_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -785,7 +753,7 @@ class SuperGlueForKeypointMatching(SuperGluePreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, KeypointMatchingOutput]: - """ + r""" Examples: ```python diff --git a/src/transformers/models/superpoint/modeling_superpoint.py b/src/transformers/models/superpoint/modeling_superpoint.py index 219a0e52ad..58dd7ade98 100644 --- a/src/transformers/models/superpoint/modeling_superpoint.py +++ b/src/transformers/models/superpoint/modeling_superpoint.py @@ -27,18 +27,13 @@ from transformers.models.superpoint.configuration_superpoint import SuperPointCo from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, ) logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "SuperPointConfig" - -_CHECKPOINT_FOR_DOC = "magic-leap-community/superpoint" - def remove_keypoints_from_borders( keypoints: torch.Tensor, scores: torch.Tensor, border: int, height: int, width: int @@ -323,12 +318,8 @@ class SuperPointDescriptorDecoder(nn.Module): return descriptors +@auto_docstring class SuperPointPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SuperPointConfig base_model_prefix = "superpoint" main_input_name = "pixel_values" @@ -363,33 +354,10 @@ class SuperPointPreTrainedModel(PreTrainedModel): return pixel_values[:, 0, :, :][:, None, :, :] -SUPERPOINT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SuperPointConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +@auto_docstring( + custom_intro=""" + SuperPoint model outputting keypoints and descriptors. """ - -SUPERPOINT_INPUTS_DOCSTRING = r""" -Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`SuperPointImageProcessor`]. See - [`SuperPointImageProcessor.__call__`] for details. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more - detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - """ - - -@add_start_docstrings( - "SuperPoint model outputting keypoints and descriptors.", - SUPERPOINT_START_DOCSTRING, ) class SuperPointForKeypointDetection(SuperPointPreTrainedModel): """ @@ -412,7 +380,7 @@ class SuperPointForKeypointDetection(SuperPointPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(SUPERPOINT_INPUTS_DOCSTRING) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -420,7 +388,7 @@ class SuperPointForKeypointDetection(SuperPointPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, SuperPointKeypointDescriptionOutput]: - """ + r""" Examples: ```python diff --git a/src/transformers/models/swiftformer/modeling_swiftformer.py b/src/transformers/models/swiftformer/modeling_swiftformer.py index 5870e91338..71b7ab8d96 100644 --- a/src/transformers/models/swiftformer/modeling_swiftformer.py +++ b/src/transformers/models/swiftformer/modeling_swiftformer.py @@ -23,33 +23,14 @@ from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2CLS -from ...modeling_outputs import ( - BaseModelOutputWithNoAttention, - ImageClassifierOutputWithNoAttention, -) +from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import auto_docstring, logging from .configuration_swiftformer import SwiftFormerConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "SwiftFormerConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "MBZUAI/swiftformer-xs" -_EXPECTED_OUTPUT_SHAPE = [1, 220, 7, 7] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "MBZUAI/swiftformer-xs" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - class SwiftFormerPatchEmbedding(nn.Module): """ @@ -405,12 +386,8 @@ class SwiftFormerEncoder(nn.Module): ) +@auto_docstring class SwiftFormerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SwiftFormerConfig base_model_prefix = "swiftformer" main_input_name = "pixel_values" @@ -428,35 +405,7 @@ class SwiftFormerPreTrainedModel(PreTrainedModel): nn.init.constant_(module.weight, 1.0) -SWIFTFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SwiftFormerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SWIFTFORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare SwiftFormer Model transformer outputting raw hidden-states without any specific head on top.", - SWIFTFORMER_START_DOCSTRING, -) +@auto_docstring class SwiftFormerModel(SwiftFormerPreTrainedModel): def __init__(self, config: SwiftFormerConfig): super().__init__(config) @@ -468,22 +417,13 @@ class SwiftFormerModel(SwiftFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWIFTFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithNoAttention]: - r""" """ - output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -508,12 +448,7 @@ class SwiftFormerModel(SwiftFormerPreTrainedModel): ) -@add_start_docstrings( - """ - SwiftFormer Model transformer with an image classification head on top (e.g. for ImageNet). - """, - SWIFTFORMER_START_DOCSTRING, -) +@auto_docstring class SwiftFormerForImageClassification(SwiftFormerPreTrainedModel): def __init__(self, config: SwiftFormerConfig) -> None: super().__init__(config) @@ -531,13 +466,7 @@ class SwiftFormerForImageClassification(SwiftFormerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWIFTFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutputWithNoAttention, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/swin/modeling_swin.py b/src/transformers/models/swin/modeling_swin.py index ad998f6ff6..31373ec370 100644 --- a/src/transformers/models/swin/modeling_swin.py +++ b/src/transformers/models/swin/modeling_swin.py @@ -28,32 +28,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, meshgrid, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from ...utils.backbone_utils import BackboneMixin from .configuration_swin import SwinConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "SwinConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/swin-tiny-patch4-window7-224" -_EXPECTED_OUTPUT_SHAPE = [1, 49, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "microsoft/swin-tiny-patch4-window7-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" - # drop_path, SwinPatchEmbeddings, SwinPatchMerging and SwinDropPath are from the timm library. @@ -919,12 +900,8 @@ class SwinEncoder(nn.Module): ) +@auto_docstring class SwinPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SwinConfig base_model_prefix = "swin" main_input_name = "pixel_values" @@ -951,53 +928,15 @@ class SwinPreTrainedModel(PreTrainedModel): module.relative_position_bias_table.data.zero_() -SWIN_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`SwinConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SWIN_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Swin Model transformer outputting raw hidden-states without any specific head on top.", - SWIN_START_DOCSTRING, - """ - add_pooling_layer (`bool`, *optional*, defaults to `True`): - Whether or not to apply pooling layer. - use_mask_token (`bool`, *optional*, defaults to `False`): - Whether or not to create and apply mask tokens in the embedding layer. - """, -) +@auto_docstring class SwinModel(SwinPreTrainedModel): def __init__(self, config, add_pooling_layer=True, use_mask_token=False): + r""" + add_pooling_layer (`bool`, *optional*, defaults to `True`): + Whether or not to apply pooling layer. + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether or not to create and apply mask tokens in the embedding layer. + """ super().__init__(config) self.config = config self.num_layers = len(config.depths) @@ -1023,14 +962,7 @@ class SwinModel(SwinPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SWIN_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SwinModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1096,8 +1028,9 @@ class SwinModel(SwinPreTrainedModel): ) -@add_start_docstrings( - """Swin Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886). +@auto_docstring( + custom_intro=""" + Swin Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886). @@ -1105,8 +1038,7 @@ class SwinModel(SwinPreTrainedModel): directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - """, - SWIN_START_DOCSTRING, + """ ) class SwinForMaskedImageModeling(SwinPreTrainedModel): def __init__(self, config): @@ -1125,8 +1057,7 @@ class SwinForMaskedImageModeling(SwinPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWIN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SwinMaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1141,8 +1072,6 @@ class SwinForMaskedImageModeling(SwinPreTrainedModel): bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, SwinForMaskedImageModeling @@ -1214,8 +1143,8 @@ class SwinForMaskedImageModeling(SwinPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Swin Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. @@ -1226,8 +1155,7 @@ class SwinForMaskedImageModeling(SwinPreTrainedModel): position embeddings to the higher resolution. - """, - SWIN_START_DOCSTRING, + """ ) class SwinForImageClassification(SwinPreTrainedModel): def __init__(self, config): @@ -1244,13 +1172,7 @@ class SwinForImageClassification(SwinPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWIN_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=SwinImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1299,11 +1221,10 @@ class SwinForImageClassification(SwinPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Swin backbone, to be used with frameworks like DETR and MaskFormer. - """, - SWIN_START_DOCSTRING, + """ ) class SwinBackbone(SwinPreTrainedModel, BackboneMixin): def __init__(self, config: SwinConfig): diff --git a/src/transformers/models/swin2sr/image_processing_swin2sr_fast.py b/src/transformers/models/swin2sr/image_processing_swin2sr_fast.py index 9dd056af1e..7b8e6843e6 100644 --- a/src/transformers/models/swin2sr/image_processing_swin2sr_fast.py +++ b/src/transformers/models/swin2sr/image_processing_swin2sr_fast.py @@ -16,14 +16,8 @@ from typing import List, Optional, Union -from ...image_processing_utils import ( - BatchFeature, - ChannelDimension, - get_image_size, -) +from ...image_processing_utils import BatchFeature, ChannelDimension, get_image_size from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -33,7 +27,7 @@ from ...image_utils import ImageInput from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -51,20 +45,18 @@ if is_torchvision_available(): class Swin2SRFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + do_pad (`bool`, *optional*, defaults to `True`): + Whether to pad the image to make the height and width divisible by `window_size`. + pad_size (`int`, *optional*, defaults to `8`): + The size of the sliding window for the local attention. + """ + do_pad: Optional[bool] pad_size: Optional[int] -@add_start_docstrings( - "Constructs a fast Swin2SR image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - do_pad (`bool`, *optional*, defaults to `True`): - Whether to pad the image to make the height and width divisible by `window_size`. - pad_size (`int`, *optional*, defaults to `8`): - The size of the sliding window for the local attention. - """, -) +@auto_docstring class Swin2SRImageProcessorFast(BaseImageProcessorFast): do_rescale = True rescale_factor = 1 / 255 @@ -75,15 +67,6 @@ class Swin2SRImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[Swin2SRFastImageProcessorKwargs]): super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - do_pad (`bool`, *optional*, defaults to `True`): - Whether to pad the image to make the height and width divisible by `window_size`. - pad_size (`int`, *optional*, defaults to `8`): - The size of the sliding window for the local attention. - """, - ) def preprocess(self, images: ImageInput, **kwargs: Unpack[Swin2SRFastImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) diff --git a/src/transformers/models/swin2sr/modeling_swin2sr.py b/src/transformers/models/swin2sr/modeling_swin2sr.py index 6543fe5cdf..44e12cfe02 100644 --- a/src/transformers/models/swin2sr/modeling_swin2sr.py +++ b/src/transformers/models/swin2sr/modeling_swin2sr.py @@ -27,26 +27,12 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, ImageSuperResolutionOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, meshgrid, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_swin2sr import Swin2SRConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "Swin2SRConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "caidas/swin2SR-classical-sr-x2-64" -_EXPECTED_OUTPUT_SHAPE = [1, 180, 488, 648] - @dataclass class Swin2SREncoderOutput(ModelOutput): @@ -748,12 +734,8 @@ class Swin2SREncoder(nn.Module): ) +@auto_docstring class Swin2SRPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Swin2SRConfig base_model_prefix = "swin2sr" main_input_name = "pixel_values" @@ -770,43 +752,7 @@ class Swin2SRPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -SWIN2SR_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Swin2SRConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SWIN2SR_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`Swin2SRImageProcessor.__call__`] for details. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Swin2SR Model transformer outputting raw hidden-states without any specific head on top.", - SWIN2SR_START_DOCSTRING, -) +@auto_docstring class Swin2SRModel(Swin2SRPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -857,14 +803,7 @@ class Swin2SRModel(Swin2SRPreTrainedModel): return pixel_values - @add_start_docstrings_to_model_forward(SWIN2SR_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1058,11 +997,10 @@ class PixelShuffleAuxUpsampler(nn.Module): return reconstruction, aux -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Swin2SR Model transformer with an upsampler head on top for image super resolution and restoration. - """, - SWIN2SR_START_DOCSTRING, + """ ) class Swin2SRForImageSuperResolution(Swin2SRPreTrainedModel): def __init__(self, config): @@ -1091,8 +1029,7 @@ class Swin2SRForImageSuperResolution(Swin2SRPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWIN2SR_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageSuperResolutionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1103,8 +1040,6 @@ class Swin2SRForImageSuperResolution(Swin2SRPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, ImageSuperResolutionOutput]: r""" - Returns: - Example: ```python >>> import torch diff --git a/src/transformers/models/swinv2/modeling_swinv2.py b/src/transformers/models/swinv2/modeling_swinv2.py index 52e33cf3d9..ea340d730c 100644 --- a/src/transformers/models/swinv2/modeling_swinv2.py +++ b/src/transformers/models/swinv2/modeling_swinv2.py @@ -28,32 +28,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, meshgrid, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from ...utils.backbone_utils import BackboneMixin from .configuration_swinv2 import Swinv2Config logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "Swinv2Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/swinv2-tiny-patch4-window8-256" -_EXPECTED_OUTPUT_SHAPE = [1, 64, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "microsoft/swinv2-tiny-patch4-window8-256" -_IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" - # drop_path, Swinv2PatchEmbeddings, Swinv2PatchMerging and Swinv2DropPath are from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/swin_transformer_v2.py. @@ -975,12 +956,8 @@ class Swinv2Encoder(nn.Module): ) +@auto_docstring class Swinv2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Swinv2Config base_model_prefix = "swinv2" main_input_name = "pixel_values" @@ -1007,48 +984,16 @@ class Swinv2PreTrainedModel(PreTrainedModel): module.logit_scale.data.fill_(math.log(10)) -SWINV2_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Swinv2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SWINV2_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, default `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Swinv2 Model transformer outputting raw hidden-states without any specific head on top.", - SWINV2_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.swin.modeling_swin.SwinModel with SWIN->SWINV2,Swin->Swinv2 class Swinv2Model(Swinv2PreTrainedModel): def __init__(self, config, add_pooling_layer=True, use_mask_token=False): + r""" + add_pooling_layer (`bool`, *optional*, defaults to `True`): + Whether or not to apply pooling layer. + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether or not to create and apply mask tokens in the embedding layer. + """ super().__init__(config) self.config = config self.num_layers = len(config.depths) @@ -1074,14 +1019,7 @@ class Swinv2Model(Swinv2PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SWINV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Swinv2ModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1147,18 +1085,18 @@ class Swinv2Model(Swinv2PreTrainedModel): ) -@add_start_docstrings( - """Swinv2 Model with a decoder on top for masked image modeling, as proposed in -[SimMIM](https://arxiv.org/abs/2111.09886). +@auto_docstring( + custom_intro=""" + Swinv2 Model with a decoder on top for masked image modeling, as proposed in + [SimMIM](https://arxiv.org/abs/2111.09886). - + - Note that we provide a script to pre-train this model on custom data in our [examples - directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). + Note that we provide a script to pre-train this model on custom data in our [examples + directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - - """, - SWINV2_START_DOCSTRING, + + """ ) # Copied from transformers.models.swin.modeling_swin.SwinForMaskedImageModeling with swin->swinv2, base-simmim-window6-192->tiny-patch4-window8-256,SWIN->SWINV2,Swin->Swinv2,192->256 class Swinv2ForMaskedImageModeling(Swinv2PreTrainedModel): @@ -1178,8 +1116,7 @@ class Swinv2ForMaskedImageModeling(Swinv2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWINV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Swinv2MaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1194,8 +1131,6 @@ class Swinv2ForMaskedImageModeling(Swinv2PreTrainedModel): bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, Swinv2ForMaskedImageModeling @@ -1267,8 +1202,8 @@ class Swinv2ForMaskedImageModeling(Swinv2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Swinv2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. @@ -1279,8 +1214,7 @@ class Swinv2ForMaskedImageModeling(Swinv2PreTrainedModel): position embeddings to the higher resolution. - """, - SWINV2_START_DOCSTRING, + """ ) # Copied from transformers.models.swin.modeling_swin.SwinForImageClassification with SWIN->SWINV2,Swin->Swinv2,swin->swinv2 class Swinv2ForImageClassification(Swinv2PreTrainedModel): @@ -1298,13 +1232,7 @@ class Swinv2ForImageClassification(Swinv2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(SWINV2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=Swinv2ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1353,11 +1281,10 @@ class Swinv2ForImageClassification(Swinv2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Swinv2 backbone, to be used with frameworks like DETR and MaskFormer. - """, - SWINV2_START_DOCSTRING, + """ ) class Swinv2Backbone(Swinv2PreTrainedModel, BackboneMixin): def __init__(self, config): @@ -1374,8 +1301,7 @@ class Swinv2Backbone(Swinv2PreTrainedModel, BackboneMixin): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @add_start_docstrings_to_model_forward(SWINV2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, @@ -1383,9 +1309,7 @@ class Swinv2Backbone(Swinv2PreTrainedModel, BackboneMixin): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/switch_transformers/modeling_switch_transformers.py b/src/transformers/models/switch_transformers/modeling_switch_transformers.py index 041bd244c6..a2253026ea 100644 --- a/src/transformers/models/switch_transformers/modeling_switch_transformers.py +++ b/src/transformers/models/switch_transformers/modeling_switch_transformers.py @@ -38,13 +38,11 @@ from ...pytorch_utils import ALL_LAYERNORM_LAYERS, find_pruneable_heads_and_indi from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from .configuration_switch_transformers import SwitchTransformersConfig @@ -57,8 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "SwitchTransformersConfig" -_CHECKPOINT_FOR_DOC = "google/switch-base-8" #################################################### # This dict contains ids and associated url @@ -766,12 +762,8 @@ class SwitchTransformersBlock(nn.Module): return outputs # hidden-states, past_key_value, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights), (router_tuple) +@auto_docstring class SwitchTransformersPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = SwitchTransformersConfig base_model_prefix = "switch_transformers" supports_gradient_checkpointing = True @@ -1257,171 +1249,6 @@ class SwitchTransformersStack(SwitchTransformersPreTrainedModel): return causal_mask -SWITCH_TRANSFORMERS_START_DOCSTRING = r""" - - The SWITCH_TRANSFORMERS model was proposed in [Switch Transformers: Scaling to Trillion Parameter Models with - Simple and Efficient Sparsity](https://arxiv.org/abs/2101.03961) by [William - Fedus](https://arxiv.org/search/cs?searchtype=author&query=Fedus%2C+W), [Barret - Zoph](https://arxiv.org/search/cs?searchtype=author&query=Zoph%2C+B), and [Noam - Shazeer](https://arxiv.org/search/cs?searchtype=author&query=Shazeer%2C+N). It's an encoder-decoder T5-like model - with sparse Feed Forward that stands for Mixture of Experts (MoE) architecture. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`SwitchTransformersConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -SWITCH_TRANSFORMERS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position - embeddings so you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - [What are input IDs?](../glossary#input-ids) - - To know more on how to prepare `input_ids` for pretraining take a look a [SWITCH_TRANSFORMERS - Training](./switch_transformers#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - SWITCH_TRANSFORMERS uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [SWITCH_TRANSFORMERS - Training](./switch_transformers#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - -SWITCH_TRANSFORMERS_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position - embeddings so you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - To know more on how to prepare `input_ids` for pretraining take a look a [SWITCH_TRANSFORMERS - Training](./switch_transformers#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - output_router_logits (`bool`, *optional*): - Whether or not to return the logits of all the routers. They are useful for computing the router loss, and - should not be returned during inference. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask __HEAD_MASK_WARNING_MSG = """ The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently, @@ -1431,10 +1258,7 @@ num_heads)`. """ -@add_start_docstrings( - "The bare SWITCH_TRANSFORMERS Model transformer outputting raw hidden-states without any specific head on top.", - SWITCH_TRANSFORMERS_START_DOCSTRING, -) +@auto_docstring class SwitchTransformersModel(SwitchTransformersPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1486,8 +1310,7 @@ class SwitchTransformersModel(SwitchTransformersPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SWITCH_TRANSFORMERS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqMoEModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1509,7 +1332,46 @@ class SwitchTransformersModel(SwitchTransformersPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqMoEModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position + embeddings so you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [SWITCH_TRANSFORMERS + Training](./switch_transformers#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + SWITCH_TRANSFORMERS uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [SWITCH_TRANSFORMERS + Training](./switch_transformers#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1607,8 +1469,10 @@ class SwitchTransformersModel(SwitchTransformersPreTrainedModel): ) -@add_start_docstrings( - """SWITCH_TRANSFORMERS Model with a `language modeling` head on top.""", SWITCH_TRANSFORMERS_START_DOCSTRING +@auto_docstring( + custom_intro=""" + SWITCH_TRANSFORMERS Model with a `language modeling` head on top. + """ ) class SwitchTransformersForConditionalGeneration(SwitchTransformersPreTrainedModel, GenerationMixin): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"] @@ -1667,8 +1531,7 @@ class SwitchTransformersForConditionalGeneration(SwitchTransformersPreTrainedMod def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(SWITCH_TRANSFORMERS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqMoEOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1691,13 +1554,51 @@ class SwitchTransformersForConditionalGeneration(SwitchTransformersPreTrainedMod cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqMoEOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position + embeddings so you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [SWITCH_TRANSFORMERS + Training](./switch_transformers#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + SWITCH_TRANSFORMERS uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [SWITCH_TRANSFORMERS + Training](./switch_transformers#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -1893,10 +1794,10 @@ class SwitchTransformersForConditionalGeneration(SwitchTransformersPreTrainedMod return reordered_decoder_past -@add_start_docstrings( - "The bare SWITCH_TRANSFORMERS Model transformer outputting encoder's raw hidden-states without any specific head" - " on top.", - SWITCH_TRANSFORMERS_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare SWITCH_TRANSFORMERS Model transformer outputting encoder's raw hidden-states without any specific head + """ ) class SwitchTransformersEncoderModel(SwitchTransformersPreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight"] @@ -1938,8 +1839,7 @@ class SwitchTransformersEncoderModel(SwitchTransformersPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.block[layer].layer[0].SelfAttention.prune_heads(heads) - @add_start_docstrings_to_model_forward(SWITCH_TRANSFORMERS_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MoEModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1952,7 +1852,15 @@ class SwitchTransformersEncoderModel(SwitchTransformersPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], MoEModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. SWITCH_TRANSFORMERS is a model with relative position + embeddings so you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + To know more on how to prepare `input_ids` for pretraining take a look a [SWITCH_TRANSFORMERS + Training](./switch_transformers#training). Example: diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index ec520e8791..da8fe7db17 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -43,12 +43,11 @@ from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from ...utils.model_parallel_utils import assert_device_map, get_device_map from .configuration_t5 import T5Config @@ -62,8 +61,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "T5Config" -_CHECKPOINT_FOR_DOC = "google-t5/t5-small" #################################################### # This dict contains ids and associated url @@ -768,12 +765,8 @@ class T5ClassificationHead(nn.Module): return hidden_states +@auto_docstring class T5PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = T5Config load_tf_weights = load_tf_weights_in_t5 base_model_prefix = "transformer" @@ -1328,160 +1321,6 @@ class T5Stack(T5PreTrainedModel): return causal_mask -T5_START_DOCSTRING = r""" - - The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text - Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan - Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a - text-to-text denoising generative setting. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`T5Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -T5_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you - should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - [What are input IDs?](../glossary#input-ids) - - To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 - Training](./t5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - -T5_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you - should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask __HEAD_MASK_WARNING_MSG = """ The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently, @@ -1491,10 +1330,7 @@ num_heads)`. """ -@add_start_docstrings( - "The bare T5 Model transformer outputting raw hidden-states without any specific head on top.", - T5_START_DOCSTRING, -) +@auto_docstring class T5Model(T5PreTrainedModel): _keys_to_ignore_on_load_unexpected = [ "decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight", @@ -1584,8 +1420,7 @@ class T5Model(T5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(T5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1606,7 +1441,44 @@ class T5Model(T5PreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1701,7 +1573,11 @@ class T5Model(T5PreTrainedModel): ) -@add_start_docstrings("""T5 Model with a `language modeling` head on top.""", T5_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + T5 Model with a `language modeling` head on top. + """ +) class T5ForConditionalGeneration(T5PreTrainedModel, GenerationMixin): _keys_to_ignore_on_load_unexpected = [ "decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight", @@ -1795,8 +1671,7 @@ class T5ForConditionalGeneration(T5PreTrainedModel, GenerationMixin): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(T5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1818,13 +1693,49 @@ class T5ForConditionalGeneration(T5PreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -1986,10 +1897,7 @@ class T5ForConditionalGeneration(T5PreTrainedModel, GenerationMixin): return reordered_decoder_past -@add_start_docstrings( - "The bare T5 Model transformer outputting encoder's raw hidden-states without any specific head on top.", - T5_START_DOCSTRING, -) +@auto_docstring class T5EncoderModel(T5PreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight"] _keys_to_ignore_on_load_unexpected = [r"decoder"] @@ -2062,8 +1970,7 @@ class T5EncoderModel(T5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.block[layer].layer[0].SelfAttention.prune_heads(heads) - @add_start_docstrings_to_model_forward(T5_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2075,7 +1982,14 @@ class T5EncoderModel(T5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). Example: @@ -2105,12 +2019,11 @@ class T5EncoderModel(T5PreTrainedModel): return encoder_outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" T5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - T5_START_DOCSTRING, + """ ) class T5ForSequenceClassification(T5PreTrainedModel): _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"] @@ -2126,8 +2039,7 @@ class T5ForSequenceClassification(T5PreTrainedModel): self.model_parallel = False - @add_start_docstrings_to_model_forward(T5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2147,10 +2059,47 @@ class T5ForSequenceClassification(T5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -2238,13 +2187,7 @@ class T5ForSequenceClassification(T5PreTrainedModel): ) -@add_start_docstrings( - """ - T5 Encoder Model with a token classification head on top (a linear layer on top of the hidden-states output) - e.g. for Named-Entity-Recognition (NER) tasks. - """, - T5_START_DOCSTRING, -) +@auto_docstring class T5ForTokenClassification(T5PreTrainedModel): _tied_weights_keys = ["transformer.encoder.embed_tokens.weight"] @@ -2259,8 +2202,7 @@ class T5ForTokenClassification(T5PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(T5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2273,9 +2215,18 @@ class T5ForTokenClassification(T5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2310,13 +2261,7 @@ class T5ForTokenClassification(T5PreTrainedModel): ) -@add_start_docstrings( - """ - T5 Model with a span classification head on top for extractive question-answering tasks like SQuAD (linear layers - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - T5_START_DOCSTRING, -) +@auto_docstring class T5ForQuestionAnswering(T5PreTrainedModel): _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"] _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -2366,8 +2311,7 @@ class T5ForQuestionAnswering(T5PreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(T5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -2388,15 +2332,44 @@ class T5ForQuestionAnswering(T5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict use_cache = use_cache if use_cache is not None else self.config.use_cache diff --git a/src/transformers/models/table_transformer/modeling_table_transformer.py b/src/transformers/models/table_transformer/modeling_table_transformer.py index 3c25697c39..7793331bf2 100644 --- a/src/transformers/models/table_transformer/modeling_table_transformer.py +++ b/src/transformers/models/table_transformer/modeling_table_transformer.py @@ -27,11 +27,9 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttenti from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_timm_available, logging, - replace_return_docstrings, requires_backends, ) from ...utils.backbone_utils import load_backbone @@ -44,9 +42,6 @@ if is_timm_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "TableTransformerConfig" -_CHECKPOINT_FOR_DOC = "microsoft/table-transformer-detection" - @dataclass # Copied from transformers.models.detr.modeling_detr.DetrDecoderOutput with DETR->TABLE_TRANSFORMER,Detr->TableTransformer @@ -730,6 +725,7 @@ class TableTransformerDecoderLayer(nn.Module): return outputs +@auto_docstring class TableTransformerPreTrainedModel(PreTrainedModel): config_class = TableTransformerConfig base_model_prefix = "model" @@ -758,60 +754,6 @@ class TableTransformerPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -TABLE_TRANSFORMER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TableTransformerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -TABLE_TRANSFORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. - - Pixel values can be obtained using [`DetrImageProcessor`]. See [`DetrImageProcessor.__call__`] for details. - - pixel_mask (`torch.FloatTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - - [What are attention masks?](../glossary#attention-mask) - - decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): - Not used by default. Can be used to mask object queries. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you - can choose to directly pass a flattened representation of an image. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): - Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an - embedded representation. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class TableTransformerEncoder(TableTransformerPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1105,12 +1047,11 @@ class TableTransformerDecoder(TableTransformerPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The bare Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) outputting raw hidden-states without any specific head on top. - """, - TABLE_TRANSFORMER_START_DOCSTRING, + """ ) class TableTransformerModel(TableTransformerPreTrainedModel): # Copied from transformers.models.detr.modeling_detr.DetrModel.__init__ with Detr->TableTransformer @@ -1147,8 +1088,7 @@ class TableTransformerModel(TableTransformerPreTrainedModel): for name, param in self.backbone.conv_encoder.model.named_parameters(): param.requires_grad_(True) - @add_start_docstrings_to_model_forward(TABLE_TRANSFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TableTransformerModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1162,7 +1102,14 @@ class TableTransformerModel(TableTransformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], TableTransformerModelOutput]: r""" - Returns: + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. Examples: @@ -1274,12 +1221,11 @@ class TableTransformerModel(TableTransformerPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) with object detection heads on top, for tasks such as COCO detection. - """, - TABLE_TRANSFORMER_START_DOCSTRING, + """ ) class TableTransformerForObjectDetection(TableTransformerPreTrainedModel): # Copied from transformers.models.detr.modeling_detr.DetrForObjectDetection.__init__ with Detr->TableTransformer @@ -1300,8 +1246,7 @@ class TableTransformerForObjectDetection(TableTransformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(TABLE_TRANSFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TableTransformerObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1316,14 +1261,20 @@ class TableTransformerForObjectDetection(TableTransformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], TableTransformerObjectDetectionOutput]: r""" + decoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*): + Not used by default. Can be used to mask object queries. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you + can choose to directly pass a flattened representation of an image. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`, *optional*): + Optionally, instead of initializing the queries with a tensor of zeros, you can choose to directly pass an + embedded representation. labels (`List[Dict]` of len `(batch_size,)`, *optional*): Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py index b54d5be1ef..35295ebf88 100644 --- a/src/transformers/models/tapas/modeling_tapas.py +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -28,26 +28,13 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, MaskedLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import ( - apply_chunking_to_forward, - find_pruneable_heads_and_indices, - prune_linear_layer, -) -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer +from ...utils import ModelOutput, auto_docstring, logging from .configuration_tapas import TapasConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "TapasConfig" -_CHECKPOINT_FOR_DOC = "google/tapas-base" - EPSILON_ZERO_DIVISION = 1e-10 CLOSE_ENOUGH_TO_LOG_ZERO = -10000.0 @@ -708,12 +695,8 @@ class TapasOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class TapasPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = TapasConfig base_model_prefix = "tapas" supports_gradient_checkpointing = True @@ -739,68 +722,7 @@ class TapasPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -TAPAS_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its models (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TapasConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -TAPAS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0}, 7)`, *optional*): - Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this - class for more info. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. If - `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be - used. Selected in the range `[0, config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 - indicates the head is **not masked**, - 0 indicates the head is **masked**. - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Tapas Model transformer outputting raw hidden-states without any specific head on top.", - TAPAS_START_DOCSTRING, -) +@auto_docstring class TapasModel(TapasPreTrainedModel): """ This class is a small change compared to [`BertModel`], taking into account the additional token type ids. @@ -813,6 +735,10 @@ class TapasModel(TapasPreTrainedModel): """ def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -838,8 +764,7 @@ class TapasModel(TapasPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -855,7 +780,17 @@ class TapasModel(TapasPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, 7)`, *optional*): + Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this + class for more info. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. If + `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be + used. Selected in the range `[0, config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) Examples: @@ -953,7 +888,7 @@ class TapasModel(TapasPreTrainedModel): ) -@add_start_docstrings("""Tapas Model with a `language modeling` head on top.""", TAPAS_START_DOCSTRING) +@auto_docstring class TapasForMaskedLM(TapasPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] config_class = TapasConfig @@ -975,8 +910,7 @@ class TapasForMaskedLM(TapasPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -994,13 +928,22 @@ class TapasForMaskedLM(TapasPreTrainedModel): **kwargs, ) -> Union[Tuple, MaskedLMOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, 7)`, *optional*): + Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this + class for more info. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. If + `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be + used. Selected in the range `[0, config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -1063,13 +1006,12 @@ class TapasForMaskedLM(TapasPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Tapas Model with a cell selection head and optional aggregation head on top for question-answering tasks on tables (linear layers on top of the hidden-states output to compute `logits` and optional `logits_aggregation`), e.g. for SQA, WTQ or WikiSQL-supervised tasks. - """, - TAPAS_START_DOCSTRING, + """ ) class TapasForQuestionAnswering(TapasPreTrainedModel): def __init__(self, config: TapasConfig): @@ -1106,8 +1048,7 @@ class TapasForQuestionAnswering(TapasPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=TableQuestionAnsweringOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1127,6 +1068,17 @@ class TapasForQuestionAnswering(TapasPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TableQuestionAnsweringOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, 7)`, *optional*): + Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this + class for more info. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. If + `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be + used. Selected in the range `[0, config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) table_mask (`torch.LongTensor` of shape `(batch_size, seq_length)`, *optional*): Mask for the table. Indicates which tokens belong to the table (1). Question tokens, table headers and padding are 0. @@ -1136,7 +1088,6 @@ class TapasForQuestionAnswering(TapasPreTrainedModel): - 1 for tokens that are **part of the answer**, - 0 for tokens that are **not part of the answer**. - aggregation_labels (`torch.LongTensor` of shape `(batch_size, )`, *optional*): Aggregation function index for every example in the batch for computing the aggregation loss. Indices should be in `[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong supervision for @@ -1152,8 +1103,6 @@ class TapasForQuestionAnswering(TapasPreTrainedModel): Scale of the numeric values of every token. Can be obtained using [`AutoTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss. - Returns: - Examples: ```python @@ -1409,12 +1358,11 @@ class TapasForQuestionAnswering(TapasPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Tapas Model with a sequence classification head on top (a linear layer on top of the pooled output), e.g. for table entailment tasks, such as TabFact (Chen et al., 2020). - """, - TAPAS_START_DOCSTRING, + """ ) class TapasForSequenceClassification(TapasPreTrainedModel): def __init__(self, config): @@ -1428,8 +1376,7 @@ class TapasForSequenceClassification(TapasPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1444,14 +1391,23 @@ class TapasForSequenceClassification(TapasPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, 7)`, *optional*): + Token indices that encode tabular structure. Indices can be obtained using [`AutoTokenizer`]. See this + class for more info. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. If + `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be + used. Selected in the range `[0, config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Note: this is called "classification_class_index" in the original implementation. - Returns: - Examples: ```python diff --git a/src/transformers/models/textnet/modeling_textnet.py b/src/transformers/models/textnet/modeling_textnet.py index d32a7351af..6fd5e15d90 100644 --- a/src/transformers/models/textnet/modeling_textnet.py +++ b/src/transformers/models/textnet/modeling_textnet.py @@ -21,7 +21,7 @@ import torch.nn as nn from torch import Tensor from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss -from transformers import PreTrainedModel, add_start_docstrings +from transformers import PreTrainedModel from transformers.activations import ACT2CLS from transformers.modeling_outputs import ( BackboneOutput, @@ -30,22 +30,14 @@ from transformers.modeling_outputs import ( ImageClassifierOutputWithNoAttention, ) from transformers.models.textnet.configuration_textnet import TextNetConfig -from transformers.utils import ( - add_code_sample_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from transformers.utils import logging from transformers.utils.backbone_utils import BackboneMixin +from ...utils import auto_docstring + logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "TextNetConfig" -_CHECKPOINT_FOR_DOC = "czczup/textnet-base" -_EXPECTED_OUTPUT_SHAPE = [1, 512, 20, 27] - class TextNetConvLayer(nn.Module): def __init__(self, config: TextNetConfig): @@ -224,37 +216,8 @@ class TextNetEncoder(nn.Module): return BaseModelOutputWithNoAttention(last_hidden_state=hidden_state, hidden_states=hidden_states) -TEXTNET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`TextNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -TEXTNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`TextNetImageProcessor.__call__`] for details. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - +@auto_docstring class TextNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = TextNetConfig base_model_prefix = "textnet" main_input_name = "pixel_values" @@ -270,10 +233,7 @@ class TextNetPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -@add_start_docstrings( - "The bare Textnet model outputting raw features without any specific head on top.", - TEXTNET_START_DOCSTRING, -) +@auto_docstring class TextNetModel(TextNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -282,14 +242,7 @@ class TextNetModel(TextNetPreTrainedModel): self.pooler = nn.AdaptiveAvgPool2d((2, 2)) self.post_init() - @add_start_docstrings_to_model_forward(TEXTNET_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndNoAttention, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> Union[Tuple[Any, List[Any]], Tuple[Any], BaseModelOutputWithPoolingAndNoAttention]: @@ -318,12 +271,11 @@ class TextNetModel(TextNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" TextNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. - """, - TEXTNET_START_DOCSTRING, + """ ) class TextNetForImageClassification(TextNetPreTrainedModel): def __init__(self, config): @@ -340,8 +292,7 @@ class TextNetForImageClassification(TextNetPreTrainedModel): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(TEXTNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -355,8 +306,6 @@ class TextNetForImageClassification(TextNetPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python >>> import torch @@ -413,11 +362,10 @@ class TextNetForImageClassification(TextNetPreTrainedModel): return ImageClassifierOutputWithNoAttention(loss=loss, logits=logits, hidden_states=outputs.hidden_states) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" TextNet backbone, to be used with frameworks like DETR and MaskFormer. - """, - TEXTNET_START_DOCSTRING, + """ ) class TextNetBackbone(TextNetPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -430,14 +378,11 @@ class TextNetBackbone(TextNetPreTrainedModel, BackboneMixin): # initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(TEXTNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None ) -> Union[Tuple[Tuple], BackboneOutput]: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py b/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py index d63be6d8d7..6ba0d88e94 100644 --- a/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py +++ b/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py @@ -33,18 +33,14 @@ from ...modeling_outputs import ( from ...modeling_utils import PreTrainedModel from ...time_series_utils import NegativeBinomialOutput, NormalOutput, StudentTOutput from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from .configuration_time_series_transformer import TimeSeriesTransformerConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "TimeSeriesTransformerConfig" - class TimeSeriesFeatureEmbedder(nn.Module): """ @@ -625,6 +621,7 @@ class TimeSeriesTransformerDecoderLayer(nn.Module): return outputs +@auto_docstring class TimeSeriesTransformerPreTrainedModel(PreTrainedModel): config_class = TimeSeriesTransformerConfig base_model_prefix = "model" @@ -645,177 +642,6 @@ class TimeSeriesTransformerPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -TIME_SERIES_TRANSFORMER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TimeSeriesTransformerConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -TIME_SERIES_TRANSFORMER_INPUTS_DOCSTRING = r""" - Args: - past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`): - Past values of the time series, that serve as context in order to predict the future. The sequence size of - this tensor must be larger than the `context_length` of the model, since the model will use the larger size - to construct lag features, i.e. additional values from the past which are added in order to serve as "extra - context". - - The `sequence_length` here is equal to `config.context_length` + `max(config.lags_sequence)`, which if no - `lags_sequence` is configured, is equal to `config.context_length` + 7 (as by default, the largest - look-back index in `config.lags_sequence` is 7). The property `_past_length` returns the actual length of - the past. - - The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as - `static_categorical_features`, `static_real_features`, `past_time_features` and lags). - - Optionally, missing values need to be replaced with zeros and indicated via the `past_observed_mask`. - - For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of - variates in the time series per time step. - past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`): - Required time features, which the model internally will add to `past_values`. These could be things like - "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These - could also be so-called "age" features, which basically help the model know "at which point in life" a - time-series is. Age features have small values for distant past time steps and increase monotonically the - more we approach the current time step. Holiday features are also a good example of time features. - - These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where - the position encodings are learned from scratch internally as parameters of the model, the Time Series - Transformer requires to provide additional time features. The Time Series Transformer only learns - additional embeddings for `static_categorical_features`. - - Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features - must but known at prediction time. - - The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. - past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): - Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in - `[0, 1]`: - - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - - static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): - Optional static categorical features for which the model will learn an embedding, which it will add to the - values of the time series. - - Static categorical features are features which have the same value for all time steps (static over time). - - A typical example of a static categorical feature is a time series ID. - static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): - Optional static real features which the model will add to the values of the time series. - - Static real features are features which have the same value for all time steps (static over time). - - A typical example of a static real feature is promotion information. - future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)` or `(batch_size, prediction_length, input_size)`, *optional*): - Future values of the time series, that serve as labels for the model. The `future_values` is what the - Transformer needs during training to learn to output, given the `past_values`. - - The sequence length here is equal to `prediction_length`. - - See the demo notebook and code snippets for details. - - Optionally, during training any missing values need to be replaced with zeros and indicated via the - `future_observed_mask`. - - For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of - variates in the time series per time step. - future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`): - Required time features for the prediction window, which the model internally will add to `future_values`. - These could be things like "month of year", "day of the month", etc. encoded as vectors (for instance as - Fourier features). These could also be so-called "age" features, which basically help the model know "at - which point in life" a time-series is. Age features have small values for distant past time steps and - increase monotonically the more we approach the current time step. Holiday features are also a good example - of time features. - - These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where - the position encodings are learned from scratch internally as parameters of the model, the Time Series - Transformer requires to provide additional time features. The Time Series Transformer only learns - additional embeddings for `static_categorical_features`. - - Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features - must but known at prediction time. - - The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. - future_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): - Boolean mask to indicate which `future_values` were observed and which were missing. Mask values selected - in `[0, 1]`: - - - 1 for values that are **observed**, - - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). - - This mask is used to filter out missing values for the final loss calculation. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on certain token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Mask to avoid performing attention on certain token indices. By default, a causal mask will be used, to - make sure the model can only look at previous inputs in order to predict the future. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class TimeSeriesTransformerEncoder(TimeSeriesTransformerPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1170,10 +996,7 @@ class TimeSeriesTransformerDecoder(TimeSeriesTransformerPreTrainedModel): ) -@add_start_docstrings( - "The bare Time Series Transformer Model outputting raw hidden-states without any specific head on top.", - TIME_SERIES_TRANSFORMER_START_DOCSTRING, -) +@auto_docstring class TimeSeriesTransformerModel(TimeSeriesTransformerPreTrainedModel): def __init__(self, config: TimeSeriesTransformerConfig): super().__init__(config) @@ -1312,8 +1135,7 @@ class TimeSeriesTransformerModel(TimeSeriesTransformerPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(TIME_SERIES_TRANSFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqTSModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1335,7 +1157,98 @@ class TimeSeriesTransformerModel(TimeSeriesTransformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Seq2SeqTSModelOutput, Tuple]: r""" - Returns: + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`): + Past values of the time series, that serve as context in order to predict the future. The sequence size of + this tensor must be larger than the `context_length` of the model, since the model will use the larger size + to construct lag features, i.e. additional values from the past which are added in order to serve as "extra + context". + + The `sequence_length` here is equal to `config.context_length` + `max(config.lags_sequence)`, which if no + `lags_sequence` is configured, is equal to `config.context_length` + 7 (as by default, the largest + look-back index in `config.lags_sequence` is 7). The property `_past_length` returns the actual length of + the past. + + The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as + `static_categorical_features`, `static_real_features`, `past_time_features` and lags). + + Optionally, missing values need to be replaced with zeros and indicated via the `past_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`): + Required time features, which the model internally will add to `past_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. Holiday features are also a good example of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in + `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): + Optional static categorical features for which the model will learn an embedding, which it will add to the + values of the time series. + + Static categorical features are features which have the same value for all time steps (static over time). + + A typical example of a static categorical feature is a time series ID. + static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): + Optional static real features which the model will add to the values of the time series. + + Static real features are features which have the same value for all time steps (static over time). + + A typical example of a static real feature is promotion information. + future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)` or `(batch_size, prediction_length, input_size)`, *optional*): + Future values of the time series, that serve as labels for the model. The `future_values` is what the + Transformer needs during training to learn to output, given the `past_values`. + + The sequence length here is equal to `prediction_length`. + + See the demo notebook and code snippets for details. + + Optionally, during training any missing values need to be replaced with zeros and indicated via the + `future_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`): + Required time features for the prediction window, which the model internally will add to `future_values`. + These could be things like "month of year", "day of the month", etc. encoded as vectors (for instance as + Fourier features). These could also be so-called "age" features, which basically help the model know "at + which point in life" a time-series is. Age features have small values for distant past time steps and + increase monotonically the more we approach the current time step. Holiday features are also a good example + of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. Examples: @@ -1431,10 +1344,7 @@ class TimeSeriesTransformerModel(TimeSeriesTransformerPreTrainedModel): ) -@add_start_docstrings( - "The Time Series Transformer Model with a distribution head on top for time-series forecasting.", - TIME_SERIES_TRANSFORMER_START_DOCSTRING, -) +@auto_docstring class TimeSeriesTransformerForPrediction(TimeSeriesTransformerPreTrainedModel): def __init__(self, config: TimeSeriesTransformerConfig): super().__init__(config) @@ -1475,8 +1385,7 @@ class TimeSeriesTransformerForPrediction(TimeSeriesTransformerPreTrainedModel): sliced_params = [p[:, -trailing_n:] for p in params] return self.distribution_output.distribution(sliced_params, loc=loc, scale=scale) - @add_start_docstrings_to_model_forward(TIME_SERIES_TRANSFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqTSModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -1499,7 +1408,106 @@ class TimeSeriesTransformerForPrediction(TimeSeriesTransformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Seq2SeqTSModelOutput, Tuple]: r""" - Returns: + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`): + Past values of the time series, that serve as context in order to predict the future. The sequence size of + this tensor must be larger than the `context_length` of the model, since the model will use the larger size + to construct lag features, i.e. additional values from the past which are added in order to serve as "extra + context". + + The `sequence_length` here is equal to `config.context_length` + `max(config.lags_sequence)`, which if no + `lags_sequence` is configured, is equal to `config.context_length` + 7 (as by default, the largest + look-back index in `config.lags_sequence` is 7). The property `_past_length` returns the actual length of + the past. + + The `past_values` is what the Transformer encoder gets as input (with optional additional features, such as + `static_categorical_features`, `static_real_features`, `past_time_features` and lags). + + Optionally, missing values need to be replaced with zeros and indicated via the `past_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + past_time_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_features)`): + Required time features, which the model internally will add to `past_values`. These could be things like + "month of year", "day of the month", etc. encoded as vectors (for instance as Fourier features). These + could also be so-called "age" features, which basically help the model know "at which point in life" a + time-series is. Age features have small values for distant past time steps and increase monotonically the + more we approach the current time step. Holiday features are also a good example of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected in + `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + static_categorical_features (`torch.LongTensor` of shape `(batch_size, number of static categorical features)`, *optional*): + Optional static categorical features for which the model will learn an embedding, which it will add to the + values of the time series. + + Static categorical features are features which have the same value for all time steps (static over time). + + A typical example of a static categorical feature is a time series ID. + static_real_features (`torch.FloatTensor` of shape `(batch_size, number of static real features)`, *optional*): + Optional static real features which the model will add to the values of the time series. + + Static real features are features which have the same value for all time steps (static over time). + + A typical example of a static real feature is promotion information. + future_values (`torch.FloatTensor` of shape `(batch_size, prediction_length)` or `(batch_size, prediction_length, input_size)`, *optional*): + Future values of the time series, that serve as labels for the model. The `future_values` is what the + Transformer needs during training to learn to output, given the `past_values`. + + The sequence length here is equal to `prediction_length`. + + See the demo notebook and code snippets for details. + + Optionally, during training any missing values need to be replaced with zeros and indicated via the + `future_observed_mask`. + + For multivariate time series, the `input_size` > 1 dimension is required and corresponds to the number of + variates in the time series per time step. + future_time_features (`torch.FloatTensor` of shape `(batch_size, prediction_length, num_features)`): + Required time features for the prediction window, which the model internally will add to `future_values`. + These could be things like "month of year", "day of the month", etc. encoded as vectors (for instance as + Fourier features). These could also be so-called "age" features, which basically help the model know "at + which point in life" a time-series is. Age features have small values for distant past time steps and + increase monotonically the more we approach the current time step. Holiday features are also a good example + of time features. + + These features serve as the "positional encodings" of the inputs. So contrary to a model like BERT, where + the position encodings are learned from scratch internally as parameters of the model, the Time Series + Transformer requires to provide additional time features. The Time Series Transformer only learns + additional embeddings for `static_categorical_features`. + + Additional dynamic real covariates can be concatenated to this tensor, with the caveat that these features + must but known at prediction time. + + The `num_features` here is equal to `config.`num_time_features` + `config.num_dynamic_real_features`. + future_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, input_size)`, *optional*): + Boolean mask to indicate which `future_values` were observed and which were missing. Mask values selected + in `[0, 1]`: + + - 1 for values that are **observed**, + - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros). + + This mask is used to filter out missing values for the final loss calculation. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of `last_hidden_state`, `hidden_states` (*optional*) and `attentions` (*optional*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` (*optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. Examples: diff --git a/src/transformers/models/timesfm/modeling_timesfm.py b/src/transformers/models/timesfm/modeling_timesfm.py index e34f823500..b54f7c15b8 100644 --- a/src/transformers/models/timesfm/modeling_timesfm.py +++ b/src/transformers/models/timesfm/modeling_timesfm.py @@ -32,18 +32,11 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, logging from .configuration_timesfm import TimesFmConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "TimesFmConfig" @dataclass @@ -309,30 +302,8 @@ class TimesFmDecoderLayer(nn.Module): return scores, hidden_states -TIMESFM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TimesFmConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare TimesFM Model outputting raw hidden-states without any specific head on top.", - TIMESFM_START_DOCSTRING, -) +@auto_docstring class TimesFmPreTrainedModel(PreTrainedModel): - """handles the loading for all models.""" - config_class = TimesFmConfig base_model_prefix = "timesfm" _no_split_modules = ["TimesFmDecoderLayer"] @@ -360,28 +331,8 @@ class TimesFmPreTrainedModel(PreTrainedModel): nn.init.ones_(module.scaling) -TIMESFM_INPUTS_DOCSTRING = r""" - Args: - past_values: list of time series forecast contexts. Each context time series - can be a torch Tensor of potentially different context lengths. - freq: frequency of each context time series in the inputs. 0 for high frequency - (default), 1 for medium, and 2 for low. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. -""" - - -@add_start_docstrings( - "The bare TimesFM Model outputting raw hidden-states without any specific head on top.", - TIMESFM_START_DOCSTRING, -) +@auto_docstring class TimesFmModel(TimesFmPreTrainedModel): - """Patched time-series decoder without any specific output layer.""" - def __init__(self, config: TimesFmConfig): super().__init__(config) @@ -422,7 +373,7 @@ class TimesFmModel(TimesFmPreTrainedModel): return outputs, (mu, sigma) @can_return_tuple - @add_start_docstrings_to_model_forward(TIMESFM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -431,9 +382,13 @@ class TimesFmModel(TimesFmPreTrainedModel): output_attentions: bool = False, output_hidden_states: bool = False, ) -> TimesFmOutput: - """ + r""" past_values_padding (`torch.LongTensor` of shape `(batch_size, sequence_length)`): The padding indicator of the time series. + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Past values of the time series that serves as input to the model. + freq (`torch.LongTensor` of shape `(batch_size,)`): + Frequency indices for the time series data. """ # Reshape into patches (using view for efficiency) bsize = past_values.shape[0] @@ -729,8 +684,7 @@ class TimesFmModelForPrediction(TimesFmPreTrainedModel): return torch.stack(losses).mean() @can_return_tuple - @add_start_docstrings_to_model_forward(TIMESFM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TimesFmOutputForPrediction, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: Sequence[torch.Tensor], @@ -744,23 +698,25 @@ class TimesFmModelForPrediction(TimesFmPreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> TimesFmOutputForPrediction: r""" - window_size (`int`, *optional*): - Window size of trend + residual decomposition. If None then we do not do decomposition. - future_values (`torch.Tensor`, *optional*): - Optional future time series values to be used for loss computation. - forecast_context_len (`int`, *optional*): - Optional max context length. - return_forecast_on_context (`bool`, *optional*): - True to return the forecast on the context when available, i.e. after the first input patch. - truncate_negative (`bool`, *optional*): - Truncate to only non-negative values if any of the contexts have non-negative values, - otherwise do nothing. - output_attentions (`bool`, *optional*): - Whether to output the attentions. - output_hidden_states (`bool`, *optional*): - Whether to output the hidden states. - - Returns: + window_size (`int`, *optional*): + Window size of trend + residual decomposition. If None then we do not do decomposition. + future_values (`torch.Tensor`, *optional*): + Optional future time series values to be used for loss computation. + forecast_context_len (`int`, *optional*): + Optional max context length. + return_forecast_on_context (`bool`, *optional*): + True to return the forecast on the context when available, i.e. after the first input patch. + truncate_negative (`bool`, *optional*): + Truncate to only non-negative values if any of the contexts have non-negative values, + otherwise do nothing. + output_attentions (`bool`, *optional*): + Whether to output the attentions. + output_hidden_states (`bool`, *optional*): + Whether to output the hidden states. + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Past values of the time series that serves as input to the model. + freq (`torch.LongTensor` of shape `(batch_size,)`): + Frequency indices for the time series data. Example: diff --git a/src/transformers/models/timesfm/modular_timesfm.py b/src/transformers/models/timesfm/modular_timesfm.py index e285a4a19e..20ee49c01a 100644 --- a/src/transformers/models/timesfm/modular_timesfm.py +++ b/src/transformers/models/timesfm/modular_timesfm.py @@ -26,13 +26,7 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, logging from ..llama.modeling_llama import LlamaRMSNorm from ..phi4_multimodal.modeling_phi4_multimodal import simple_eager_attention_forward from .configuration_timesfm import TimesFmConfig @@ -40,9 +34,6 @@ from .configuration_timesfm import TimesFmConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/timesfm-2.0-500m-pytorch" -_CONFIG_FOR_DOC = "TimesFmConfig" - @dataclass class TimesFmOutput(BaseModelOutput): @@ -267,30 +258,8 @@ class TimesFmDecoderLayer(nn.Module): return scores, hidden_states -TIMESFM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TimesFmConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare TimesFM Model outputting raw hidden-states without any specific head on top.", - TIMESFM_START_DOCSTRING, -) +@auto_docstring class TimesFmPreTrainedModel(PreTrainedModel): - """handles the loading for all models.""" - config_class = TimesFmConfig base_model_prefix = "timesfm" _no_split_modules = ["TimesFmDecoderLayer"] @@ -318,28 +287,8 @@ class TimesFmPreTrainedModel(PreTrainedModel): nn.init.ones_(module.scaling) -TIMESFM_INPUTS_DOCSTRING = r""" - Args: - past_values: list of time series forecast contexts. Each context time series - can be a torch Tensor of potentially different context lengths. - freq: frequency of each context time series in the inputs. 0 for high frequency - (default), 1 for medium, and 2 for low. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. -""" - - -@add_start_docstrings( - "The bare TimesFM Model outputting raw hidden-states without any specific head on top.", - TIMESFM_START_DOCSTRING, -) +@auto_docstring class TimesFmModel(TimesFmPreTrainedModel): - """Patched time-series decoder without any specific output layer.""" - def __init__(self, config: TimesFmConfig): super().__init__(config) @@ -380,7 +329,7 @@ class TimesFmModel(TimesFmPreTrainedModel): return outputs, (mu, sigma) @can_return_tuple - @add_start_docstrings_to_model_forward(TIMESFM_INPUTS_DOCSTRING) + @auto_docstring def forward( self, past_values: torch.Tensor, @@ -389,9 +338,13 @@ class TimesFmModel(TimesFmPreTrainedModel): output_attentions: bool = False, output_hidden_states: bool = False, ) -> TimesFmOutput: - """ + r""" past_values_padding (`torch.LongTensor` of shape `(batch_size, sequence_length)`): The padding indicator of the time series. + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Past values of the time series that serves as input to the model. + freq (`torch.LongTensor` of shape `(batch_size,)`): + Frequency indices for the time series data. """ # Reshape into patches (using view for efficiency) bsize = past_values.shape[0] @@ -687,8 +640,7 @@ class TimesFmModelForPrediction(TimesFmPreTrainedModel): return torch.stack(losses).mean() @can_return_tuple - @add_start_docstrings_to_model_forward(TIMESFM_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TimesFmOutputForPrediction, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, past_values: Sequence[torch.Tensor], @@ -702,23 +654,25 @@ class TimesFmModelForPrediction(TimesFmPreTrainedModel): output_hidden_states: Optional[bool] = None, ) -> TimesFmOutputForPrediction: r""" - window_size (`int`, *optional*): - Window size of trend + residual decomposition. If None then we do not do decomposition. - future_values (`torch.Tensor`, *optional*): - Optional future time series values to be used for loss computation. - forecast_context_len (`int`, *optional*): - Optional max context length. - return_forecast_on_context (`bool`, *optional*): - True to return the forecast on the context when available, i.e. after the first input patch. - truncate_negative (`bool`, *optional*): - Truncate to only non-negative values if any of the contexts have non-negative values, - otherwise do nothing. - output_attentions (`bool`, *optional*): - Whether to output the attentions. - output_hidden_states (`bool`, *optional*): - Whether to output the hidden states. - - Returns: + window_size (`int`, *optional*): + Window size of trend + residual decomposition. If None then we do not do decomposition. + future_values (`torch.Tensor`, *optional*): + Optional future time series values to be used for loss computation. + forecast_context_len (`int`, *optional*): + Optional max context length. + return_forecast_on_context (`bool`, *optional*): + True to return the forecast on the context when available, i.e. after the first input patch. + truncate_negative (`bool`, *optional*): + Truncate to only non-negative values if any of the contexts have non-negative values, + otherwise do nothing. + output_attentions (`bool`, *optional*): + Whether to output the attentions. + output_hidden_states (`bool`, *optional*): + Whether to output the hidden states. + past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Past values of the time series that serves as input to the model. + freq (`torch.LongTensor` of shape `(batch_size,)`): + Frequency indices for the time series data. Example: diff --git a/src/transformers/models/timesformer/modeling_timesformer.py b/src/transformers/models/timesformer/modeling_timesformer.py index 0bd79a6cec..31454a7108 100644 --- a/src/transformers/models/timesformer/modeling_timesformer.py +++ b/src/transformers/models/timesformer/modeling_timesformer.py @@ -26,15 +26,15 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, ImageClassifierOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import ( + auto_docstring, + logging, +) from .configuration_timesformer import TimesformerConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "TimesformerConfig" -_CHECKPOINT_FOR_DOC = "facebook/timesformer" - # Adapted from https://github.com/facebookresearch/TimeSformer/blob/a5ef29a7b7264baff199a30b3306ac27de901133/timesformer/models/vit.py#L155 class TimesformerPatchEmbeddings(nn.Module): @@ -458,12 +458,8 @@ class TimesformerEncoder(nn.Module): ) +@auto_docstring class TimesformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = TimesformerConfig base_model_prefix = "timesformer" main_input_name = "pixel_values" @@ -484,38 +480,7 @@ class TimesformerPreTrainedModel(PreTrainedModel): module.patch_embeddings.apply(self._init_weights) -TIMESFORMER_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`TimesformerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -TIMESFORMER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`VideoMAEImageProcessor.preprocess`] for details. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare TimeSformer Model transformer outputting raw hidden-states without any specific head on top.", - TIMESFORMER_START_DOCSTRING, -) +@auto_docstring class TimesformerModel(TimesformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -540,8 +505,7 @@ class TimesformerModel(TimesformerPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(TIMESFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -550,8 +514,6 @@ class TimesformerModel(TimesformerPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutput]: r""" - Returns: - Examples: ```python @@ -653,10 +615,11 @@ class TimesformerModel(TimesformerPreTrainedModel): ) -@add_start_docstrings( - """TimeSformer Model transformer with a video classification head on top (a linear layer on top of the final hidden state -of the [CLS] token) e.g. for ImageNet.""", - TIMESFORMER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + TimeSformer Model transformer with a video classification head on top (a linear layer on top of the final hidden state + of the [CLS] token) e.g. for ImageNet. + """ ) class TimesformerForVideoClassification(TimesformerPreTrainedModel): def __init__(self, config): @@ -671,8 +634,7 @@ class TimesformerForVideoClassification(TimesformerPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(TIMESFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -687,8 +649,6 @@ class TimesformerForVideoClassification(TimesformerPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py b/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py index 48c1d85825..28fb807ccd 100644 --- a/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py +++ b/src/transformers/models/timm_wrapper/modeling_timm_wrapper.py @@ -22,12 +22,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...modeling_outputs import ImageClassifierOutput, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings_to_model_forward, - is_timm_available, - replace_return_docstrings, - requires_backends, -) +from ...utils import auto_docstring, is_timm_available, requires_backends from .configuration_timm_wrapper import TimmWrapperConfig @@ -61,22 +56,7 @@ class TimmWrapperModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor, ...]] = None -TIMM_WRAPPER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`TimmWrapperImageProcessor.preprocess`] - for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. Not compatible with timm wrapped models. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. Not compatible with timm wrapped models. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - **kwargs: - Additional keyword arguments passed along to the `timm` model forward. -""" - - +@auto_docstring class TimmWrapperPreTrainedModel(PreTrainedModel): main_input_name = "pixel_values" config_class = TimmWrapperConfig @@ -139,8 +119,7 @@ class TimmWrapperModel(TimmWrapperPreTrainedModel): self.timm_model = timm.create_model(config.architecture, pretrained=False, num_classes=0) self.post_init() - @add_start_docstrings_to_model_forward(TIMM_WRAPPER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TimmWrapperModelOutput, config_class=TimmWrapperConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -151,12 +130,16 @@ class TimmWrapperModel(TimmWrapperPreTrainedModel): **kwargs, ) -> Union[TimmWrapperModelOutput, Tuple[Tensor, ...]]: r""" + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. Not compatible with timm wrapped models. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. Not compatible with timm wrapped models. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. do_pooling (`bool`, *optional*): Whether to do pooling for the last_hidden_state in `TimmWrapperModel` or not. If `None` is passed, the `do_pooling` value from the config is used. - Returns: - Examples: ```python >>> import torch @@ -254,8 +237,7 @@ class TimmWrapperForImageClassification(TimmWrapperPreTrainedModel): self.num_labels = config.num_labels self.post_init() - @add_start_docstrings_to_model_forward(TIMM_WRAPPER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=TimmWrapperConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -270,8 +252,14 @@ class TimmWrapperForImageClassification(TimmWrapperPreTrainedModel): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - - Returns: + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. Not compatible with timm wrapped models. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. Not compatible with timm wrapped models. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + **kwargs: + Additional keyword arguments passed along to the `timm` model forward. Examples: ```python diff --git a/src/transformers/models/trocr/modeling_trocr.py b/src/transformers/models/trocr/modeling_trocr.py index fcc0de5b6d..e2c77f5779 100644 --- a/src/transformers/models/trocr/modeling_trocr.py +++ b/src/transformers/models/trocr/modeling_trocr.py @@ -27,15 +27,12 @@ from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _prepare_4d_attention_mask, _prepare_4d_causal_attention_mask from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_trocr import TrOCRConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "TrOCRConfig" -_CHECKPOINT_FOR_DOC = "microsoft/trocr-base-handwritten" - # Copied from transformers.models.bart.modeling_bart.BartLearnedPositionalEmbedding with Bart->TrOCR class TrOCRLearnedPositionalEmbedding(nn.Embedding): @@ -414,6 +411,7 @@ class TrOCRDecoderLayer(nn.Module): return outputs +@auto_docstring class TrOCRPreTrainedModel(PreTrainedModel): config_class = TrOCRConfig base_model_prefix = "model" @@ -432,23 +430,6 @@ class TrOCRPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -TROCR_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`TrOCRConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - class TrOCRDecoder(TrOCRPreTrainedModel): """ Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TrOCRDecoderLayer`] @@ -714,16 +695,14 @@ class TrOCRDecoder(TrOCRPreTrainedModel): ) -@add_start_docstrings( - "The TrOCR Model with a language modeling head. Can be used for summarization.", - TROCR_START_DOCSTRING, -) -class TrOCRDecoderWrapper(TrOCRPreTrainedModel): - """ +@auto_docstring( + custom_intro=""" + The TrOCR Model with a language modeling head. Can be used for summarization. This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the [`EncoderDecoderModel`] framework. """ - +) +class TrOCRDecoderWrapper(TrOCRPreTrainedModel): def __init__(self, config): super().__init__(config) self.decoder = TrOCRDecoder(config) @@ -732,10 +711,10 @@ class TrOCRDecoderWrapper(TrOCRPreTrainedModel): return self.decoder(*args, **kwargs) -@add_start_docstrings( - "The TrOCR Decoder with a language modeling head. Can be used as the decoder part of [`EncoderDecoderModel`] and" - " [`VisionEncoderDecoder`].", - TROCR_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The TrOCR Decoder with a language modeling head. Can be used as the decoder part of [`EncoderDecoderModel`] and + """ ) class TrOCRForCausalLM(TrOCRPreTrainedModel, GenerationMixin): _tied_weights_keys = ["output_projection.weight"] @@ -770,7 +749,7 @@ class TrOCRForCausalLM(TrOCRPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -788,72 +767,15 @@ class TrOCRForCausalLM(TrOCRPreTrainedModel, GenerationMixin): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/tvp/modeling_tvp.py b/src/transformers/models/tvp/modeling_tvp.py index d7e2058152..02ff2c8c79 100644 --- a/src/transformers/models/tvp/modeling_tvp.py +++ b/src/transformers/models/tvp/modeling_tvp.py @@ -23,11 +23,10 @@ import torch.utils.checkpoint from torch import nn from ...activations import ACT2FN -from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import prune_linear_layer -from ...utils import logging +from ...utils import auto_docstring, logging from ...utils.backbone_utils import load_backbone from .configuration_tvp import TvpConfig @@ -562,11 +561,8 @@ class TvpPooler(nn.Module): return pooled_output +@auto_docstring class TvpPreTrainedModel(PreTrainedModel): - """An abstract class to handle weights initialization and - a simple interface for downloading and loading pretrained models. - """ - config_class = TvpConfig base_model_prefix = "model" supports_gradient_checkpointing = True @@ -590,55 +586,6 @@ class TvpPreTrainedModel(PreTrainedModel): nn.init.constant_(module.bias, 0) -TVP_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`TvpConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -TVP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`TvpImageProcessor`]. See [`TvpImageProcessor.__call__`] - for details. - - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): - Whether to interpolate the pre-trained image pad prompter encodings and positional encodings. -""" - - class TvpFrameDownPadPrompter(nn.Module): """ Pad frames extracted from videos only at the bottom. @@ -772,9 +719,10 @@ TVP_PROMPTER_CLASSES_MAPPING = { } -@add_start_docstrings( - "The bare Tvp Model transformer outputting BaseModelOutputWithPooling object without any specific head on top.", - TVP_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The bare Tvp Model transformer outputting BaseModelOutputWithPooling object without any specific head on top. + """ ) class TvpModel(TvpPreTrainedModel): def __init__(self, config): @@ -806,8 +754,7 @@ class TvpModel(TvpPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(TVP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=TvpConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -820,8 +767,6 @@ class TvpModel(TvpPreTrainedModel): interpolate_pos_encoding: bool = False, ): r""" - Returns: - Examples: ```python >>> import torch @@ -897,11 +842,10 @@ class TvpVideoGroundingHead(nn.Module): return logits -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Tvp Model with a video grounding head on top computing IoU, distance, and duration loss. - """, - TVP_START_DOCSTRING, + """ ) class TvpForVideoGrounding(TvpPreTrainedModel): def __init__(self, config): @@ -912,8 +856,7 @@ class TvpForVideoGrounding(TvpPreTrainedModel): self.post_init() - @add_start_docstrings_to_model_forward(TVP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TvpVideoGroundingOutput, config_class=TvpConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -929,7 +872,6 @@ class TvpForVideoGrounding(TvpPreTrainedModel): r""" labels (`torch.FloatTensor` of shape `(batch_size, 3)`, *optional*): The labels contains duration, start time, and end time of the video corresponding to the text. - Returns: Examples: ```python diff --git a/src/transformers/models/udop/modeling_udop.py b/src/transformers/models/udop/modeling_udop.py index c5af44a952..b1049851fb 100644 --- a/src/transformers/models/udop/modeling_udop.py +++ b/src/transformers/models/udop/modeling_udop.py @@ -41,11 +41,9 @@ from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torchdynamo_compiling, - replace_return_docstrings, ) @@ -58,174 +56,6 @@ if is_torch_flex_attn_available(): logger = logging.getLogger(__name__) -_CONFIG_FOR_DOC = "UdopConfig" - - -UDOP_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Args: - config ([`UdopConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -UDOP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. UDOP is a model with relative position embeddings so - you should be able to pad the inputs on both the right and the left. Indices can be obtained using - [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail. - [What are input IDs?](../glossary#input-ids) - - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Batch of document images. Each image is divided into patches of shape `(num_channels, config.patch_size, - config.patch_size)` and the total number of patches (=`patch_sequence_length`) equals to `((height / - config.patch_size) * (width / config.patch_size))`. - - visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*): - Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model. - - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using - [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting - token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last - `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare - `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If - `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value of - `inputs_embeds`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - - -UDOP_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you - should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): - Bounding boxes of each input sequence tokens. Selected in the range `[0, - config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) - format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, - y1) represents the position of the lower right corner. - - Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] - token. See `pixel_values` for `patch_sequence_length`. - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Batch of document images. Each image is divided into patches of shape `(num_channels, config.patch_size, - config.patch_size)` and the total number of patches (=`patch_sequence_length`) equals to `((height / - config.patch_size) * (width / config.patch_size))`. - - visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*): - Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - @dataclass class BaseModelOutputWithAttentionMask(ModelOutput): """ @@ -414,12 +244,8 @@ class UdopPatchEmbeddings(nn.Module): return embeddings +@auto_docstring class UdopPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. Based on `T5PreTrainedModel`. - """ - config_class = UdopConfig base_model_prefix = "transformer" supports_gradient_checkpointing = True @@ -1660,10 +1486,7 @@ class UdopStack(UdopPreTrainedModel): return causal_mask -@add_start_docstrings( - "The bare UDOP encoder-decoder Transformer outputting raw hidden-states without any specific head on top.", - UDOP_START_DOCSTRING, -) +@auto_docstring class UdopModel(UdopPreTrainedModel): _tied_weights_keys = [ "encoder.embed_tokens.weight", @@ -1710,8 +1533,7 @@ class UdopModel(UdopPreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(UDOP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[Tensor] = None, @@ -1735,7 +1557,36 @@ class UdopModel(UdopPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Tuple[Tensor, ...]: r""" - Returns: + bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*): + Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using + [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting + token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last + `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare + `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1823,12 +1674,13 @@ class UdopModel(UdopPreTrainedModel): ) -@add_start_docstrings( - """The UDOP encoder-decoder Transformer with a language modeling head on top, enabling to generate text given document +@auto_docstring( + custom_intro=""" + The UDOP encoder-decoder Transformer with a language modeling head on top, enabling to generate text given document images and an optional prompt. - This class is based on [`T5ForConditionalGeneration`], extended to deal with images and layout (2D) data.""", - UDOP_START_DOCSTRING, + This class is based on [`T5ForConditionalGeneration`], extended to deal with images and layout (2D) data. + """ ) class UdopForConditionalGeneration(UdopPreTrainedModel, GenerationMixin): _tied_weights_keys = [ @@ -1886,8 +1738,7 @@ class UdopForConditionalGeneration(UdopPreTrainedModel, GenerationMixin): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(UDOP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[Tensor] = None, @@ -1912,13 +1763,41 @@ class UdopForConditionalGeneration(UdopPreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Tuple[Tensor, ...]: r""" + bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*): + Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using + [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. + [What are decoder input IDs?](../glossary#decoder-input-ids) T5 uses the `pad_token_id` as the starting + token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last + `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare + `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`. - Returns: - Examples: ```python @@ -2053,10 +1932,7 @@ class UdopForConditionalGeneration(UdopPreTrainedModel, GenerationMixin): return reordered_decoder_past -@add_start_docstrings( - "The bare UDOP Model transformer outputting encoder's raw hidden-states without any specific head on top.", - UDOP_START_DOCSTRING, -) +@auto_docstring class UdopEncoderModel(UdopPreTrainedModel): _tied_weights_keys = [ "encoder.embed_tokens.weight", @@ -2099,8 +1975,7 @@ class UdopEncoderModel(UdopPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.block[layer].layer[0].SelfAttention.prune_heads(heads) - @add_start_docstrings_to_model_forward(UDOP_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithAttentionMask, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[Tensor] = None, @@ -2115,7 +1990,24 @@ class UdopEncoderModel(UdopPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutputWithAttentionMask]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training). + bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): + Bounding boxes of each input sequence tokens. Selected in the range `[0, + config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) + format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, + y1) represents the position of the lower right corner. + + Note that `sequence_length = token_sequence_length + patch_sequence_length + 1` where `1` is for [CLS] + token. See `pixel_values` for `patch_sequence_length`. + visual_bbox (`torch.LongTensor` of shape `(batch_size, patch_sequence_length, 4)`, *optional*): + Bounding boxes of each patch in the image. If not provided, bounding boxes are created in the model. Example: diff --git a/src/transformers/models/umt5/modeling_umt5.py b/src/transformers/models/umt5/modeling_umt5.py index 9af07e345a..512cd159f9 100644 --- a/src/transformers/models/umt5/modeling_umt5.py +++ b/src/transformers/models/umt5/modeling_umt5.py @@ -39,13 +39,11 @@ from ...modeling_utils import PreTrainedModel from ...utils import ( DUMMY_INPUTS, DUMMY_MASK, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_torch_flex_attn_available, is_torch_fx_proxy, is_torchdynamo_compiling, logging, - replace_return_docstrings, ) from .configuration_umt5 import UMT5Config @@ -57,9 +55,6 @@ if is_torch_flex_attn_available(): logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "UMT5Config" -_CHECKPOINT_FOR_DOC = "google/umt5-small" - # Copied from transformers.models.t5.modeling_t5.T5LayerNorm with T5->UMT5 class UMT5LayerNorm(nn.Module): @@ -506,12 +501,8 @@ class UMT5ClassificationHead(nn.Module): return hidden_states +@auto_docstring class UMT5PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = UMT5Config base_model_prefix = "transformer" supports_gradient_checkpointing = True @@ -971,165 +962,7 @@ class UMT5Stack(UMT5PreTrainedModel): return causal_mask -UMT5_START_DOCSTRING = r""" - - The UMT5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text - Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan - Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a - text-to-text denoising generative setting. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`UMT5Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -UMT5_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so - you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - [What are input IDs?](../glossary#input-ids) - - To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5 - Training](./umt5#training). - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in - `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at - the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. - - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the - cache in the correct position and to infer the complete sequence length. -""" - -UMT5_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so - you should be able to pad the inputs on both the right and the left. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for detail. - - To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). - attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare UMT5 Model transformer outputting raw hidden-states without any specific head on top.", - UMT5_START_DOCSTRING, -) +@auto_docstring class UMT5Model(UMT5PreTrainedModel): r""" Examples: @@ -1204,8 +1037,7 @@ class UMT5Model(UMT5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(UMT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1226,7 +1058,44 @@ class UMT5Model(UMT5PreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5 + Training](./umt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. Example: @@ -1304,7 +1173,11 @@ class UMT5Model(UMT5PreTrainedModel): ) -@add_start_docstrings("""UMT5 Model with a `language modeling` head on top.""", UMT5_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + UMT5 Model with a `language modeling` head on top. + """ +) class UMT5ForConditionalGeneration(UMT5PreTrainedModel, GenerationMixin): r""" Examples: @@ -1380,8 +1253,7 @@ class UMT5ForConditionalGeneration(UMT5PreTrainedModel, GenerationMixin): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(UMT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1403,13 +1275,49 @@ class UMT5ForConditionalGeneration(UMT5PreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5 + Training](./umt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` - Returns: - Examples: ```python @@ -1521,10 +1429,7 @@ class UMT5ForConditionalGeneration(UMT5PreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - "The bare UMT5 Model transformer outputting encoder's raw hidden-states without any specific head on top.", - UMT5_START_DOCSTRING, -) +@auto_docstring class UMT5EncoderModel(UMT5PreTrainedModel): r""" Examples: @@ -1583,9 +1488,8 @@ class UMT5EncoderModel(UMT5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.block[layer].layer[0].SelfAttention.prune_heads(heads) - @add_start_docstrings_to_model_forward(UMT5_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) - # Copied from transformers.models.t5.modeling_t5.T5EncoderModel.forward with T5->UMT5, google-t5/t5-small->google/umt5-small + @auto_docstring + # Copied from transformers.models.t5.modeling_t5.T5EncoderModel.forward with T5->UMT5, google-t5/t5-small->google/umt5-small, t5#training->umt5#training def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1597,7 +1501,14 @@ class UMT5EncoderModel(UMT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutput]: r""" - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). Example: @@ -1627,12 +1538,11 @@ class UMT5EncoderModel(UMT5PreTrainedModel): return encoder_outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" UMT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - UMT5_START_DOCSTRING, + """ ) class UMT5ForSequenceClassification(UMT5PreTrainedModel): _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"] @@ -1649,8 +1559,7 @@ class UMT5ForSequenceClassification(UMT5PreTrainedModel): self.model_parallel = False - @add_start_docstrings_to_model_forward(UMT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1670,10 +1579,47 @@ class UMT5ForSequenceClassification(UMT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5 + Training](./umt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -1761,13 +1707,7 @@ class UMT5ForSequenceClassification(UMT5PreTrainedModel): ) -@add_start_docstrings( - """ - UMT5 Encoder Model with a token classification head on top (a linear layer on top of the hidden-states output) - e.g. for Named-Entity-Recognition (NER) tasks. - """, - UMT5_START_DOCSTRING, -) +@auto_docstring class UMT5ForTokenClassification(UMT5PreTrainedModel): _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"] _tied_weights_keys = ["transformer.encoder.embed_tokens.weight"] @@ -1784,9 +1724,8 @@ class UMT5ForTokenClassification(UMT5PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(UMT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) - # Copied from transformers.models.t5.modeling_t5.T5ForTokenClassification.forward with T5->UMT5 + @auto_docstring + # Copied from transformers.models.t5.modeling_t5.T5ForTokenClassification.forward with T5->UMT5, t5->umt5 def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1799,9 +1738,18 @@ class UMT5ForTokenClassification(UMT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you + should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1836,13 +1784,7 @@ class UMT5ForTokenClassification(UMT5PreTrainedModel): ) -@add_start_docstrings( - """ - UMT5 Model with a span classification head on top for extractive question-answering tasks like SQuAD (linear layers - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - UMT5_START_DOCSTRING, -) +@auto_docstring class UMT5ForQuestionAnswering(UMT5PreTrainedModel): _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"] @@ -1894,8 +1836,7 @@ class UMT5ForQuestionAnswering(UMT5PreTrainedModel): def get_decoder(self): return self.decoder - @add_start_docstrings_to_model_forward(UMT5_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqQuestionAnsweringModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1916,15 +1857,44 @@ class UMT5ForQuestionAnswering(UMT5PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqQuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - Returns: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so + you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training). + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5 + Training](./umt5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict use_cache = use_cache if use_cache is not None else self.config.use_cache diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index 74608797ab..48f351e21d 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -26,13 +26,7 @@ from ...modeling_outputs import ( Wav2Vec2BaseModelOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from .configuration_unispeech import UniSpeechConfig @@ -42,12 +36,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -# Base docstring -_CHECKPOINT_FOR_DOC = "patrickvonplaten/unispeech-large-1500h-cv-timit" - -# General docstring -_CONFIG_FOR_DOC = "UniSpeechConfig" - @dataclass class UniSpeechForPreTrainingOutput(ModelOutput): @@ -1045,12 +1033,8 @@ class UniSpeechGumbelVectorQuantizer(nn.Module): return codevectors, perplexity +@auto_docstring class UniSpeechPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = UniSpeechConfig base_model_prefix = "unispeech" main_input_name = "input_values" @@ -1241,71 +1225,10 @@ def _compute_mask_indices( return spec_aug_mask -_EXPECTED_OUTPUT_SHAPE = [1, 292, 1024] - - -UNISPEECH_START_DOCSTRING = r""" - UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled - Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, - Michael Zeng, Xuedong Huang. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`UniSpeechConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -UNISPEECH_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - UniSpeechBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top.", - UNISPEECH_START_DOCSTRING, -) +@auto_docstring class UniSpeechModel(UniSpeechPreTrainedModel): def __init__(self, config: UniSpeechConfig): super().__init__(config) @@ -1370,14 +1293,7 @@ class UniSpeechModel(UniSpeechPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=UniSpeechBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1387,6 +1303,11 @@ class UniSpeechModel(UniSpeechPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1426,8 +1347,10 @@ class UniSpeechModel(UniSpeechPreTrainedModel): ) -@add_start_docstrings( - """UniSpeech Model with a vector-quantization module and ctc loss for pre-training.""", UNISPEECH_START_DOCSTRING +@auto_docstring( + custom_intro=""" + UniSpeech Model with a vector-quantization module and ctc loss for pre-training. + """ ) class UniSpeechForPreTraining(UniSpeechPreTrainedModel): def __init__(self, config: UniSpeechConfig): @@ -1490,8 +1413,7 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): logits = logits / temperature return logits - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=UniSpeechForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1501,15 +1423,6 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechForPreTrainingOutput]: r""" - mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict - masked extracted features in *config.proj_codevector_dim* space. - sampled_negative_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_negatives)`, *optional*): - Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss. - Required input for pre-training. - - Returns: - Example: ```python @@ -1574,23 +1487,20 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): _HIDDEN_STATES_START_POSITION = 2 -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'mister quilter is the apposl of the midle classes and weare glad to welcom his gosepl'" -_CTC_EXPECTED_LOSS = 17.17 - -@add_start_docstrings( - """UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - UNISPEECH_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """ - target_lang (`str`, *optional*): - Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or - adapter..bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng' - by default. - """, ) class UniSpeechForCTC(UniSpeechPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.unispeech = UniSpeechModel(config) @@ -1661,14 +1571,7 @@ class UniSpeechForCTC(UniSpeechPreTrainedModel): for param in self.unispeech.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1740,12 +1643,11 @@ class UniSpeechForCTC(UniSpeechPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - UNISPEECH_START_DOCSTRING, + """ ) class UniSpeechForSequenceClassification(UniSpeechPreTrainedModel): def __init__(self, config): @@ -1792,13 +1694,7 @@ class UniSpeechForSequenceClassification(UniSpeechPreTrainedModel): for param in self.unispeech.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1809,6 +1705,11 @@ class UniSpeechForSequenceClassification(UniSpeechPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`UniSpeechProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/unispeech/modular_unispeech.py b/src/transformers/models/unispeech/modular_unispeech.py index 1096bc559b..9c1eae48e8 100644 --- a/src/transformers/models/unispeech/modular_unispeech.py +++ b/src/transformers/models/unispeech/modular_unispeech.py @@ -6,15 +6,9 @@ from typing import Optional, Tuple, Union import torch import torch.nn as nn -from ...modeling_outputs import CausalLMOutput, ModelOutput, SequenceClassifierOutput, Wav2Vec2BaseModelOutput +from ...modeling_outputs import ModelOutput, Wav2Vec2BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..wav2vec2.modeling_wav2vec2 import ( Wav2Vec2Encoder, Wav2Vec2EncoderStableLayerNorm, @@ -32,20 +26,6 @@ from .configuration_unispeech import UniSpeechConfig logger = logging.get_logger(__name__) -_HIDDEN_STATES_START_POSITION = 2 - -# General docstring -_CONFIG_FOR_DOC = "UniSpeechConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "patrickvonplaten/unispeech-large-1500h-cv-timit" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 1024] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'mister quilter is the apposl of the midle classes and weare glad to welcom his gosepl'" -_CTC_EXPECTED_LOSS = 17.17 - - @dataclass class UniSpeechForPreTrainingOutput(ModelOutput): """ @@ -147,12 +127,8 @@ class UniSpeechGumbelVectorQuantizer(Wav2Vec2GumbelVectorQuantizer): return codevectors, perplexity +@auto_docstring class UniSpeechPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = UniSpeechConfig base_model_prefix = "unispeech" main_input_name = "input_values" @@ -224,68 +200,9 @@ class UniSpeechPreTrainedModel(PreTrainedModel): return attention_mask -UNISPEECH_START_DOCSTRING = r""" - UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled - Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, - Michael Zeng, Xuedong Huang. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`UniSpeechConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -UNISPEECH_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - UniSpeechBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top.", - UNISPEECH_START_DOCSTRING, -) class UniSpeechModel(UniSpeechPreTrainedModel, Wav2Vec2Model): def __init__(self, config: UniSpeechConfig): UniSpeechPreTrainedModel.__init__(config) @@ -310,14 +227,6 @@ class UniSpeechModel(UniSpeechPreTrainedModel, Wav2Vec2Model): def freeze_feature_encoder(self): raise AttributeError("Not needed for UniSpeech") - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=UniSpeechBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) def forward( self, input_values: Optional[torch.Tensor], @@ -327,6 +236,11 @@ class UniSpeechModel(UniSpeechPreTrainedModel, Wav2Vec2Model): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -366,8 +280,10 @@ class UniSpeechModel(UniSpeechPreTrainedModel, Wav2Vec2Model): ) -@add_start_docstrings( - """UniSpeech Model with a vector-quantization module and ctc loss for pre-training.""", UNISPEECH_START_DOCSTRING +@auto_docstring( + custom_intro=""" + UniSpeech Model with a vector-quantization module and ctc loss for pre-training. + """ ) class UniSpeechForPreTraining(UniSpeechPreTrainedModel): def __init__(self, config: UniSpeechConfig): @@ -430,8 +346,7 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): logits = logits / temperature return logits - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=UniSpeechForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -441,15 +356,6 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechForPreTrainingOutput]: r""" - mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict - masked extracted features in *config.proj_codevector_dim* space. - sampled_negative_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_negatives)`, *optional*): - Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss. - Required input for pre-training. - - Returns: - Example: ```python @@ -512,46 +418,12 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel): ) -@add_start_docstrings( - """UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - UNISPEECH_START_DOCSTRING, - """ - target_lang (`str`, *optional*): - Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or - adapter..bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng' - by default. - """, -) class UniSpeechForCTC(Wav2Vec2ForCTC): - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) + pass -@add_start_docstrings( - """ - UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like - SUPERB Keyword Spotting. - """, - UNISPEECH_START_DOCSTRING, -) class UniSpeechForSequenceClassification(Wav2Vec2ForSequenceClassification): - @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) + pass __all__ = [ diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py index b1f8c4c346..a24d503987 100755 --- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py @@ -28,14 +28,7 @@ from ...modeling_outputs import ( XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_peft_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_peft_available, logging from .configuration_unispeech_sat import UniSpeechSatConfig @@ -45,12 +38,6 @@ if is_flash_attn_available(): logger = logging.get_logger(__name__) -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/unispeech-sat-base-100h-libri-ft" - -# General docstring -_CONFIG_FOR_DOC = "UniSpeechSatConfig" - @dataclass class UniSpeechSatForPreTrainingOutput(ModelOutput): @@ -1049,12 +1036,8 @@ class UniSpeechSatGumbelVectorQuantizer(nn.Module): return codevectors, perplexity +@auto_docstring class UniSpeechSatPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = UniSpeechSatConfig base_model_prefix = "unispeech_sat" main_input_name = "input_values" @@ -1245,70 +1228,10 @@ def _compute_mask_indices( return spec_aug_mask -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - - -UNISPEECH_SAT_START_DOCSTRING = r""" - UniSpeechSat was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech - Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael - Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`UniSpeechSatConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -UNISPEECH_SAT_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - UniSpeechSatBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare UniSpeechSat Model transformer outputting raw hidden-states without any specific head on top.", - UNISPEECH_SAT_START_DOCSTRING, -) +@auto_docstring class UniSpeechSatModel(UniSpeechSatPreTrainedModel): def __init__(self, config: UniSpeechSatConfig): super().__init__(config) @@ -1372,14 +1295,7 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=UniSpeechSatBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1389,6 +1305,11 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechSatBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1428,9 +1349,10 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel): ) -@add_start_docstrings( - """UniSpeechSat Model with a vector-quantization module and ctc loss for pre-training.""", - UNISPEECH_SAT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + UniSpeechSat Model with a vector-quantization module and ctc loss for pre-training. + """ ) class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): def __init__(self, config: UniSpeechSatConfig): @@ -1500,8 +1422,7 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): logits = logits / temperature return logits - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=UniSpeechSatForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1511,8 +1432,6 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechSatForPreTrainingOutput]: r""" - Returns: - Example: ```python @@ -1561,23 +1480,20 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): _HIDDEN_STATES_START_POSITION = 2 -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILDER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 39.88 - -@add_start_docstrings( - """UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - UNISPEECH_SAT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """ - target_lang (`str`, *optional*): - Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or - adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses - 'eng' by default. - """, ) class UniSpeechSatForCTC(UniSpeechSatPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.unispeech_sat = UniSpeechSatModel(config) @@ -1648,14 +1564,7 @@ class UniSpeechSatForCTC(UniSpeechSatPreTrainedModel): for param in self.unispeech_sat.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1727,12 +1636,11 @@ class UniSpeechSatForCTC(UniSpeechSatPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" UniSpeechSat Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - UNISPEECH_SAT_START_DOCSTRING, + """ ) class UniSpeechSatForSequenceClassification(UniSpeechSatPreTrainedModel): def __init__(self, config): @@ -1779,13 +1687,7 @@ class UniSpeechSatForSequenceClassification(UniSpeechSatPreTrainedModel): for param in self.unispeech_sat.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1796,6 +1698,11 @@ class UniSpeechSatForSequenceClassification(UniSpeechSatPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`UniSpeechSatProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1849,17 +1756,7 @@ class UniSpeechSatForSequenceClassification(UniSpeechSatPreTrainedModel): ) -# Frame class docstring -_FRAME_CLASS_CHECKPOINT = "microsoft/unispeech-sat-base-plus-sd" -_FRAME_EXPECTED_OUTPUT = [0, 0] - - -@add_start_docstrings( - """ - UniSpeechSat Model with a frame classification head on top for tasks like Speaker Diarization. - """, - UNISPEECH_SAT_START_DOCSTRING, -) +@auto_docstring class UniSpeechSatForAudioFrameClassification(UniSpeechSatPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1904,14 +1801,7 @@ class UniSpeechSatForAudioFrameClassification(UniSpeechSatPreTrainedModel): for param in self.unispeech_sat.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_FRAME_CLASS_CHECKPOINT, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_FRAME_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1922,6 +1812,11 @@ class UniSpeechSatForAudioFrameClassification(UniSpeechSatPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`UniSpeechSatProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -2021,16 +1916,10 @@ class TDNNLayer(nn.Module): return hidden_states -# Speaker Verification docstring -_XVECTOR_CHECKPOINT = "microsoft/unispeech-sat-base-plus-sv" -_XVECTOR_EXPECTED_OUTPUT = 0.97 - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" UniSpeechSat Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - UNISPEECH_SAT_START_DOCSTRING, + """ ) class UniSpeechSatForXVector(UniSpeechSatPreTrainedModel): def __init__(self, config): @@ -2094,14 +1983,7 @@ class UniSpeechSatForXVector(UniSpeechSatPreTrainedModel): return input_lengths - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_XVECTOR_CHECKPOINT, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_XVECTOR_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -2112,6 +1994,11 @@ class UniSpeechSatForXVector(UniSpeechSatPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`UniSpeechSatProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/unispeech_sat/modular_unispeech_sat.py b/src/transformers/models/unispeech_sat/modular_unispeech_sat.py index 44e566068e..22de13c3bd 100644 --- a/src/transformers/models/unispeech_sat/modular_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modular_unispeech_sat.py @@ -6,22 +6,9 @@ from typing import Optional, Tuple, Union import torch import torch.nn as nn -from ...modeling_outputs import ( - CausalLMOutput, - ModelOutput, - SequenceClassifierOutput, - TokenClassifierOutput, - Wav2Vec2BaseModelOutput, - XVectorOutput, -) +from ...modeling_outputs import ModelOutput, Wav2Vec2BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ..wav2vec2.modeling_wav2vec2 import ( Wav2Vec2Encoder, Wav2Vec2EncoderStableLayerNorm, @@ -43,25 +30,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 2 -# General docstring -_CONFIG_FOR_DOC = "UniSpeechSatConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "microsoft/unispeech-sat-base-100h-libri-ft" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILDER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 39.88 - -# Frame class docstring -_FRAME_CLASS_CHECKPOINT = "microsoft/unispeech-sat-base-plus-sd" -_FRAME_EXPECTED_OUTPUT = [0, 0] - -# Speaker Verification docstring -_XVECTOR_CHECKPOINT = "microsoft/unispeech-sat-base-plus-sv" -_XVECTOR_EXPECTED_OUTPUT = 0.97 - @dataclass class UniSpeechSatForPreTrainingOutput(ModelOutput): @@ -169,12 +137,8 @@ class UniSpeechSatGumbelVectorQuantizer(Wav2Vec2GumbelVectorQuantizer): return codevectors, perplexity +@auto_docstring class UniSpeechSatPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = UniSpeechSatConfig base_model_prefix = "unispeech_sat" main_input_name = "input_values" @@ -246,67 +210,9 @@ class UniSpeechSatPreTrainedModel(PreTrainedModel): return attention_mask -UNISPEECH_SAT_START_DOCSTRING = r""" - UniSpeechSat was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech - Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael - Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`UniSpeechSatConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -UNISPEECH_SAT_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - UniSpeechSatBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare UniSpeechSat Model transformer outputting raw hidden-states without any specific head on top.", - UNISPEECH_SAT_START_DOCSTRING, -) class UniSpeechSatModel(UniSpeechSatPreTrainedModel, Wav2Vec2Model): def __init__(self, config: UniSpeechSatConfig): UniSpeechSatPreTrainedModel.__init__(config) @@ -330,14 +236,6 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel, Wav2Vec2Model): def freeze_feature_encoder(self): raise AttributeError("Not needed for UniSpeechSat") - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=UniSpeechSatBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) def forward( self, input_values: Optional[torch.Tensor], @@ -347,6 +245,11 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel, Wav2Vec2Model): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechSatBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -386,9 +289,10 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel, Wav2Vec2Model): ) -@add_start_docstrings( - """UniSpeechSat Model with a vector-quantization module and ctc loss for pre-training.""", - UNISPEECH_SAT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + UniSpeechSat Model with a vector-quantization module and ctc loss for pre-training. + """ ) class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): def __init__(self, config: UniSpeechSatConfig): @@ -458,8 +362,7 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): logits = logits / temperature return logits - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=UniSpeechSatForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -469,8 +372,6 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, UniSpeechSatForPreTrainingOutput]: r""" - Returns: - Example: ```python @@ -517,86 +418,20 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel): ) -@add_start_docstrings( - """UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - UNISPEECH_SAT_START_DOCSTRING, - """ - target_lang (`str`, *optional*): - Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or - adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses - 'eng' by default. - """, -) class UniSpeechSatForCTC(Wav2Vec2ForCTC): - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - - -@add_start_docstrings( - """ - UniSpeechSat Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like - SUPERB Keyword Spotting. - """, - UNISPEECH_SAT_START_DOCSTRING, -) -class UniSpeechSatForSequenceClassification(Wav2Vec2ForSequenceClassification): - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - - -@add_start_docstrings( - """ - UniSpeechSat Model with a frame classification head on top for tasks like Speaker Diarization. - """, - UNISPEECH_SAT_START_DOCSTRING, -) -class UniSpeechSatForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification): - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_FRAME_CLASS_CHECKPOINT, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_FRAME_EXPECTED_OUTPUT, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - - -@add_start_docstrings( - """ - UniSpeechSat Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - UNISPEECH_SAT_START_DOCSTRING, -) -class UniSpeechSatForXVector(Wav2Vec2ForXVector): pass - @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_XVECTOR_CHECKPOINT, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_XVECTOR_EXPECTED_OUTPUT, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) + +class UniSpeechSatForSequenceClassification(Wav2Vec2ForSequenceClassification): + pass + + +class UniSpeechSatForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification): + pass + + +class UniSpeechSatForXVector(Wav2Vec2ForXVector): + pass __all__ = [ diff --git a/src/transformers/models/univnet/modeling_univnet.py b/src/transformers/models/univnet/modeling_univnet.py index d105711b5d..e000aecbc4 100644 --- a/src/transformers/models/univnet/modeling_univnet.py +++ b/src/transformers/models/univnet/modeling_univnet.py @@ -22,17 +22,12 @@ from torch import nn from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_univnet import UnivNetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "UnivNetConfig" - -_CHECKPOINT_FOR_DOC = "dg845/univnet-dev" - @dataclass class UnivNetModelOutput(ModelOutput): @@ -427,56 +422,7 @@ class UnivNetLvcBlock(nn.Module): layer.remove_weight_norm() -UNIVNET_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`UnivNetConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -UNIVNET_INPUTS_DOCSTRING = r""" - Converts a noise waveform and a conditioning spectrogram to a speech waveform. Passing a batch of log-mel - spectrograms returns a batch of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a - single, un-batched speech waveform. - - Args: - input_features (`torch.FloatTensor`): - Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, - config.num_mel_channels)`, or un-batched and of shape `(sequence_length, config.num_mel_channels)`. - noise_sequence (`torch.FloatTensor`, *optional*): - Tensor containing a noise sequence of standard Gaussian noise. Can be batched and of shape `(batch_size, - sequence_length, config.model_in_channels)`, or un-batched and of shape (sequence_length, - config.model_in_channels)`. If not supplied, will be randomly generated. - padding_mask (`torch.BoolTensor`, *optional*): - Mask indicating which parts of each sequence are padded. Mask values are selected in `[0, 1]`: - - - 1 for tokens that are **not masked** - - 0 for tokens that are **masked** - - The mask can be batched and of shape `(batch_size, sequence_length)` or un-batched and of shape - `(sequence_length,)`. - generator (`torch.Generator`, *optional*): - A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation - deterministic. - return_dict: - Whether to return a [`~utils.ModelOutput`] subclass instead of a plain tuple. -""" - - -@add_start_docstrings( - """UnivNet GAN vocoder.""", - UNIVNET_START_DOCSTRING, -) +@auto_docstring class UnivNetModel(PreTrainedModel): config_class = UnivNetConfig main_input_name = "input_features" @@ -520,8 +466,7 @@ class UnivNetModel(PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(UNIVNET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=UnivNetModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_features: torch.FloatTensor, @@ -531,7 +476,26 @@ class UnivNetModel(PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], UnivNetModelOutput]: r""" - Returns: + input_features (`torch.FloatTensor`): + Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, + config.num_mel_channels)`, or un-batched and of shape `(sequence_length, config.num_mel_channels)`. + noise_sequence (`torch.FloatTensor`, *optional*): + Tensor containing a noise sequence of standard Gaussian noise. Can be batched and of shape `(batch_size, + sequence_length, config.model_in_channels)`, or un-batched and of shape (sequence_length, + config.model_in_channels)`. If not supplied, will be randomly generated. + padding_mask (`torch.BoolTensor`, *optional*): + Mask indicating which parts of each sequence are padded. Mask values are selected in `[0, 1]`: + + - 1 for tokens that are **not masked** + - 0 for tokens that are **masked** + + The mask can be batched and of shape `(batch_size, sequence_length)` or un-batched and of shape + `(sequence_length,)`. + generator (`torch.Generator`, *optional*): + A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation + deterministic. + return_dict: + Whether to return a [`~utils.ModelOutput`] subclass instead of a plain tuple. Example: diff --git a/src/transformers/models/upernet/modeling_upernet.py b/src/transformers/models/upernet/modeling_upernet.py index 0196169c37..1e192be807 100644 --- a/src/transformers/models/upernet/modeling_upernet.py +++ b/src/transformers/models/upernet/modeling_upernet.py @@ -22,15 +22,11 @@ from torch.nn import CrossEntropyLoss from ...modeling_outputs import SemanticSegmenterOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings +from ...utils import auto_docstring from ...utils.backbone_utils import load_backbone from .configuration_upernet import UperNetConfig -# General docstring -_CONFIG_FOR_DOC = "UperNetConfig" - - class UperNetConvModule(nn.Module): """ A convolutional block that bundles conv/norm/activation layers. This block simplifies the usage of convolution @@ -267,12 +263,8 @@ class UperNetFCNHead(nn.Module): return output +@auto_docstring class UperNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = UperNetConfig main_input_name = "pixel_values" _no_split_modules = [] @@ -287,35 +279,10 @@ class UperNetPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -UPERNET_START_DOCSTRING = r""" - Parameters: - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - config ([`UperNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -UPERNET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers in case the backbone has them. See - `attentions` under returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers of the backbone. See `hidden_states` under - returned tensors for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """UperNet framework leveraging any vision backbone e.g. for ADE20k, CityScapes.""", - UPERNET_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + UperNet framework leveraging any vision backbone e.g. for ADE20k, CityScapes. + """ ) class UperNetForSemanticSegmentation(UperNetPreTrainedModel): def __init__(self, config): @@ -330,8 +297,7 @@ class UperNetForSemanticSegmentation(UperNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(UPERNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SemanticSegmenterOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -345,8 +311,6 @@ class UperNetForSemanticSegmentation(UperNetPreTrainedModel): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, UperNetForSemanticSegmentation diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index fc970dadc0..8b4718959e 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -27,23 +27,13 @@ from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import ModelOutput from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - LossKwargs, - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import LossKwargs, auto_docstring, can_return_tuple, is_torchdynamo_compiling, logging from ..auto import AutoModel from .configuration_video_llava import VideoLlavaConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "VideoLlavaConfig" - @dataclass class VideoLlavaModelOutputWithPast(ModelOutput): @@ -153,26 +143,7 @@ class VideoLlavaMultiModalProjector(nn.Module): return hidden_states -VIDEO_LLAVA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`VideoLlavaConfig`] or [`VideoLlavaVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - VIDEO_LLAVA_START_DOCSTRING, -) +@auto_docstring class VideoLlavaPreTrainedModel(PreTrainedModel): config_class = VideoLlavaConfig base_model_prefix = "" @@ -206,90 +177,10 @@ class VideoLlavaPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -VIDEO_LLAVA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values_images (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`VideoLlavaImageProcessor`] for processing images). - pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): - The tensors corresponding to the input video. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses - [`VideoLlavaImageProcessor`] for processing videos). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The VideoLlava model which consists of a vision backbone and a language model without language modeling head.""", - VIDEO_LLAVA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VideoLlava model which consists of a vision backbone and a language model without language modeling head. + """, ) class VideoLlavaModel(VideoLlavaPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -392,7 +283,7 @@ class VideoLlavaModel(VideoLlavaPreTrainedModel): return video_features, num_frames @can_return_tuple - @add_start_docstrings_to_model_forward(VIDEO_LLAVA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -411,6 +302,16 @@ class VideoLlavaModel(VideoLlavaPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Union[Tuple, VideoLlavaModelOutputWithPast]: + r""" + pixel_values_images (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses + [`VideoLlavaImageProcessor`] for processing images). + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): + The tensors corresponding to the input video. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses + [`VideoLlavaImageProcessor`] for processing videos). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -496,9 +397,10 @@ class VideoLlavaModel(VideoLlavaPreTrainedModel): class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... -@add_start_docstrings( - """The VideoLlava model which consists of a vision backbone and a language model.""", - VIDEO_LLAVA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VideoLlava model which consists of a vision backbone and a language model. + """ ) class VideoLlavaForConditionalGeneration(VideoLlavaPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -546,8 +448,7 @@ class VideoLlavaForConditionalGeneration(VideoLlavaPreTrainedModel, GenerationMi return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(VIDEO_LLAVA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=VideoLlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -569,19 +470,18 @@ class VideoLlavaForConditionalGeneration(VideoLlavaPreTrainedModel, GenerationMi **kwargs: Unpack[KwargsForCausalLM], ) -> Union[Tuple, VideoLlavaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + pixel_values_images (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses + [`VideoLlavaImageProcessor`] for processing images). + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, image_size, image_size)): + The tensors corresponding to the input video. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`VideoLlavaImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses + [`VideoLlavaImageProcessor`] for processing videos). + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: diff --git a/src/transformers/models/videomae/modeling_videomae.py b/src/transformers/models/videomae/modeling_videomae.py index 077df5bcf4..f8af7b5755 100755 --- a/src/transformers/models/videomae/modeling_videomae.py +++ b/src/transformers/models/videomae/modeling_videomae.py @@ -31,10 +31,8 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from ...utils.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD from .configuration_videomae import VideoMAEConfig @@ -42,9 +40,6 @@ from .configuration_videomae import VideoMAEConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "VideoMAEConfig" -_CHECKPOINT_FOR_DOC = "MCG-NJU/videomae-base" - @dataclass class VideoMAEDecoderOutput(ModelOutput): @@ -488,12 +483,8 @@ class VideoMAEEncoder(nn.Module): ) +@auto_docstring class VideoMAEPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VideoMAEConfig base_model_prefix = "videomae" main_input_name = "pixel_values" @@ -514,44 +505,7 @@ class VideoMAEPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -VIDEOMAE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`VideoMAEConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VIDEOMAE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`VideoMAEImageProcessor.__call__`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare VideoMAE Model transformer outputting raw hidden-states without any specific head on top.", - VIDEOMAE_START_DOCSTRING, -) +@auto_docstring class VideoMAEModel(VideoMAEPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -579,8 +533,7 @@ class VideoMAEModel(VideoMAEPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VIDEOMAE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -596,8 +549,6 @@ class VideoMAEModel(VideoMAEPreTrainedModel): batch must have the same number of masked patches. If `None`, then all patches are considered. Sequence length is `(num_frames // tubelet_size) * (image_size // patch_size) ** 2`. - Returns: - Examples: ```python @@ -775,9 +726,10 @@ class VideoMAEDecoder(nn.Module): return VideoMAEDecoderOutput(logits=logits, hidden_states=all_hidden_states, attentions=all_self_attentions) -@add_start_docstrings( - "The VideoMAE Model transformer with the decoder on top for self-supervised pre-training.", - VIDEOMAE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VideoMAE Model transformer with the decoder on top for self-supervised pre-training. + """ ) class VideoMAEForPreTraining(VideoMAEPreTrainedModel): def __init__(self, config): @@ -797,8 +749,7 @@ class VideoMAEForPreTraining(VideoMAEPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VIDEOMAE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=VideoMAEForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -814,8 +765,6 @@ class VideoMAEForPreTraining(VideoMAEPreTrainedModel): batch must have the same number of masked patches. Sequence length is `(num_frames // tubelet_size) * (image_size // patch_size) ** 2`. - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, VideoMAEForPreTraining @@ -959,10 +908,11 @@ class VideoMAEForPreTraining(VideoMAEPreTrainedModel): ) -@add_start_docstrings( - """VideoMAE Model transformer with a video classification head on top (a linear layer on top of the average pooled hidden - states of all tokens) e.g. for ImageNet.""", - VIDEOMAE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + VideoMAE Model transformer with a video classification head on top (a linear layer on top of the average pooled hidden + states of all tokens) e.g. for ImageNet. + """ ) class VideoMAEForVideoClassification(VideoMAEPreTrainedModel): def __init__(self, config): @@ -978,8 +928,7 @@ class VideoMAEForVideoClassification(VideoMAEPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VIDEOMAE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -995,8 +944,6 @@ class VideoMAEForVideoClassification(VideoMAEPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/vilt/modeling_vilt.py b/src/transformers/models/vilt/modeling_vilt.py index 5aa9e1e1cf..d690ce1e45 100755 --- a/src/transformers/models/vilt/modeling_vilt.py +++ b/src/transformers/models/vilt/modeling_vilt.py @@ -34,20 +34,13 @@ from ...modeling_outputs import ( TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import ( - find_pruneable_heads_and_indices, - meshgrid, - prune_linear_layer, -) -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...pytorch_utils import find_pruneable_heads_and_indices, meshgrid, prune_linear_layer +from ...utils import auto_docstring, logging from .configuration_vilt import ViltConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ViltConfig" -_CHECKPOINT_FOR_DOC = "dandelin/vilt-b32-mlm" - @dataclass class ViltForImagesAndTextClassificationOutput(ModelOutput): @@ -554,12 +547,8 @@ class ViltEncoder(nn.Module): ) +@auto_docstring class ViltPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ViltConfig base_model_prefix = "vilt" supports_gradient_checkpointing = True @@ -582,134 +571,13 @@ class ViltPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -VILT_START_DOCSTRING = r""" - This model is a PyTorch `torch.nn.Module `_ subclass. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ViltConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VILT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - [What are token type IDs?](../glossary#token-type-ids) - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`ViltImageProcessor.__call__`] for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - `What are attention masks? <../glossary.html#attention-mask>`__ - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): - Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `pixel_values` into patch embeddings. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -VILT_IMAGES_AND_TEXT_CLASSIFICATION_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - [What are token type IDs?](../glossary#token-type-ids) - - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_images, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`ViltImageProcessor.__call__`] for details. - - pixel_mask (`torch.LongTensor` of shape `(batch_size, num_images, height, width)`, *optional*): - Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: - - - 1 for pixels that are real (i.e. **not masked**), - - 0 for pixels that are padding (i.e. **masked**). - `What are attention masks? <../glossary.html#attention-mask>`__ - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - image_embeds (`torch.FloatTensor` of shape `(batch_size, num_images, num_patches, hidden_size)`, *optional*): - Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `pixel_values` into patch embeddings. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ViLT Model transformer outputting raw hidden-states without any specific head on top.", - VILT_START_DOCSTRING, -) +@auto_docstring class ViltModel(ViltPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -736,8 +604,7 @@ class ViltModel(ViltPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VILT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -754,7 +621,11 @@ class ViltModel(ViltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[BaseModelOutputWithPooling, Tuple[torch.FloatTensor]]: r""" - Returns: + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. + image_token_type_idx (`int`, *optional*): + - The token type ids for images. Examples: @@ -868,11 +739,10 @@ class ViltPooler(nn.Module): return pooled_output -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ViLT Model with a language modeling head on top as done during pretraining. - """, - VILT_START_DOCSTRING, + """ ) class ViltForMaskedLM(ViltPreTrainedModel): _tied_weights_keys = ["mlm_score.decoder.weight", "mlm_score.decoder.bias"] @@ -893,8 +763,7 @@ class ViltForMaskedLM(ViltPreTrainedModel): self.mlm_score.decoder = new_embeddings self.mlm_score.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(VILT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -911,13 +780,14 @@ class ViltForMaskedLM(ViltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[MaskedLMOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Labels for computing the masked language modeling loss. Indices should be in *[-100, 0, ..., config.vocab_size]* (see *input_ids* docstring) Tokens with indices set to *-100* are ignored (masked), the loss is only computed for the tokens with labels in *[0, ..., config.vocab_size]* - Returns: - Examples: ```python @@ -1047,12 +917,11 @@ class ViltMLMHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Vilt Model transformer with a classifier head on top (a linear layer on top of the final hidden state of the [CLS] token) for visual question answering, e.g. for VQAv2. - """, - VILT_START_DOCSTRING, + """ ) class ViltForQuestionAnswering(ViltPreTrainedModel): def __init__(self, config): @@ -1072,8 +941,7 @@ class ViltForQuestionAnswering(ViltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VILT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1090,13 +958,14 @@ class ViltForQuestionAnswering(ViltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[SequenceClassifierOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (`torch.FloatTensor` of shape `(batch_size, num_labels)`, *optional*): Labels for computing the visual question answering loss. This tensor must be either a one-hot encoding of all answers that are applicable for a given example in the batch, or a soft encoding indicating which answers are applicable, where 1.0 is the highest score. - Returns: - Examples: ```python @@ -1160,12 +1029,11 @@ class ViltForQuestionAnswering(ViltPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Vilt Model transformer with a classifier head on top (a linear layer on top of the final hidden state of the [CLS] token) for image-to-text or text-to-image retrieval, e.g. MSCOCO and F30K. - """, - VILT_START_DOCSTRING, + """ ) class ViltForImageAndTextRetrieval(ViltPreTrainedModel): def __init__(self, config): @@ -1179,8 +1047,7 @@ class ViltForImageAndTextRetrieval(ViltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VILT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1197,11 +1064,12 @@ class ViltForImageAndTextRetrieval(ViltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[SequenceClassifierOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels are currently not supported. - Returns: - Examples: ```python @@ -1260,11 +1128,10 @@ class ViltForImageAndTextRetrieval(ViltPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Vilt Model transformer with a classifier head on top for natural language visual reasoning, e.g. NLVR2. - """, - VILT_IMAGES_AND_TEXT_CLASSIFICATION_INPUTS_DOCSTRING, + """ ) class ViltForImagesAndTextClassification(ViltPreTrainedModel): def __init__(self, config): @@ -1285,8 +1152,7 @@ class ViltForImagesAndTextClassification(ViltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VILT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ViltForImagesAndTextClassificationOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1303,11 +1169,12 @@ class ViltForImagesAndTextClassification(ViltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[ViltForImagesAndTextClassificationOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Binary classification labels. - Returns: - Examples: ```python @@ -1401,13 +1268,7 @@ class ViltForImagesAndTextClassification(ViltPreTrainedModel): ) -@add_start_docstrings( - """ - ViLT Model with a token classification head on top (a linear layer on top of the final hidden-states of the text - tokens) e.g. for Named-Entity-Recognition (NER) tasks. - """, - VILT_START_DOCSTRING, -) +@auto_docstring class ViltForTokenClassification(ViltPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1421,8 +1282,7 @@ class ViltForTokenClassification(ViltPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VILT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TokenClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1439,10 +1299,11 @@ class ViltForTokenClassification(ViltPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[TokenClassifierOutput, Tuple[torch.FloatTensor]]: r""" + image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*): + Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `pixel_values` into patch embeddings. labels (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. - - Returns: """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index c1dd0f9941..51c4f032f4 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -29,20 +29,11 @@ from ...activations import ACT2FN from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - replace_return_docstrings, -) +from ...utils import auto_docstring, can_return_tuple, is_torchdynamo_compiling from ..auto import AutoModel from .configuration_vipllava import VipLlavaConfig -_CONFIG_FOR_DOC = "VipLlavaConfig" - - @dataclass class VipLlavaModelOutputWithPast(BaseModelOutputWithPast): """ @@ -140,27 +131,7 @@ class VipLlavaMultiModalProjector(nn.Module): return hidden_states -VIPLLAVA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`VipLlavaConfig`] or [`VipLlavaVisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare VipLlava Model outputting raw hidden-states without any specific head on top.", - VIPLLAVA_START_DOCSTRING, -) +@auto_docstring class VipLlavaPreTrainedModel(PreTrainedModel): config_class = VipLlavaConfig base_model_prefix = "" @@ -188,86 +159,10 @@ class VipLlavaPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -VIPLLAVA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): - The tensors corresponding to the input images. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`VipLlavaProcessor`] uses - [`CLIPImageProcessor`] for processing images). - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - vision_feature_layer (`Union[int, List[int]], *optional*, defaults to -2`): - The index of the layer to select the vision feature. If multiple indices are provided, - the vision feature of the corresponding indices will be concatenated to form the - vision features. - vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`): - The feature selection strategy used to select the vision feature from the vision backbone. - Can be one of `"default"` or `"full"`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - """The VipLlava model which consists of a vision backbone and a language model, without a language modeling head.""", - VIPLLAVA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VipLlava model which consists of a vision backbone and a language model, without a language modeling head. + """ ) class VipLlavaModel(VipLlavaPreTrainedModel): _checkpoint_conversion_mapping = {"language_model.model": "language_model"} @@ -312,7 +207,7 @@ class VipLlavaModel(VipLlavaPreTrainedModel): image_features = self.multi_modal_projector(image_features) return image_features - @add_start_docstrings_to_model_forward(VIPLLAVA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -329,6 +224,11 @@ class VipLlavaModel(VipLlavaPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, **lm_kwargs, ) -> Union[Tuple, VipLlavaModelOutputWithPast]: + r""" + vision_feature_layers (`Union[int, List[int]]`, *optional*): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -388,9 +288,10 @@ class VipLlavaModel(VipLlavaPreTrainedModel): return output if return_dict else output.to_tuple() -@add_start_docstrings( - """The VIPLLAVA model which consists of a vision backbone and a language model.""", - VIPLLAVA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VIPLLAVA model which consists of a vision backbone and a language model. + """ ) class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel, GenerationMixin): _checkpoint_conversion_mapping = { @@ -433,8 +334,7 @@ class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel, GenerationMixin) return self.model.multi_modal_projector @can_return_tuple - @add_start_docstrings_to_model_forward(VIPLLAVA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=VipLlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -454,20 +354,13 @@ class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel, GenerationMixin) **lm_kwargs, ) -> Union[Tuple, VipLlavaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + vision_feature_layers (`Union[int, List[int]]`, *optional*): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. Example: diff --git a/src/transformers/models/vipllava/modular_vipllava.py b/src/transformers/models/vipllava/modular_vipllava.py index 93fdecffec..57df2fd700 100644 --- a/src/transformers/models/vipllava/modular_vipllava.py +++ b/src/transformers/models/vipllava/modular_vipllava.py @@ -27,23 +27,13 @@ from transformers.models.llava.modeling_llava import ( ) from ...activations import ACT2FN -from ...utils import ( - add_start_docstrings_to_model_forward, - can_return_tuple, - is_torchdynamo_compiling, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torchdynamo_compiling, logging from .configuration_vipllava import VipLlavaConfig logger = logging.get_logger(__name__) -VIPLLAVA_INPUTS_DOCSTRING = None -_CONFIG_FOR_DOC = "VipLlavaConfig" - - class VipLlavaModelOutputWithPast(LlavaModelOutputWithPast): pass @@ -107,7 +97,7 @@ class VipLlavaModel(LlavaModel): image_features = self.multi_modal_projector(image_features) return image_features - @add_start_docstrings_to_model_forward(VIPLLAVA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: torch.LongTensor = None, @@ -124,6 +114,11 @@ class VipLlavaModel(LlavaModel): cache_position: Optional[torch.LongTensor] = None, **lm_kwargs, ) -> Union[Tuple, VipLlavaModelOutputWithPast]: + r""" + vision_feature_layers (`Union[int, List[int]]`, *optional*): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -184,10 +179,6 @@ class VipLlavaModel(LlavaModel): class VipLlavaForConditionalGeneration(LlavaForConditionalGeneration): - @can_return_tuple - @add_start_docstrings_to_model_forward(VIPLLAVA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=VipLlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) - # Ignore copy def forward( self, input_ids: torch.LongTensor = None, @@ -207,20 +198,13 @@ class VipLlavaForConditionalGeneration(LlavaForConditionalGeneration): **lm_kwargs, ) -> Union[Tuple, VipLlavaCausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + vision_feature_layers (`Union[int, List[int]]`, *optional*): + The vision feature layer, or the list of indexes of the layers to select + the vision feature. Example: diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index 69f95ff056..c82f4b6311 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -26,7 +26,7 @@ from ...configuration_utils import PretrainedConfig from ...generation import GenerationMixin from ...modeling_outputs import BaseModelOutput, Seq2SeqLMOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel, AutoModelForCausalLM from .configuration_vision_encoder_decoder import VisionEncoderDecoderConfig @@ -53,100 +53,8 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "VisionEncoderDecoderConfig" -VISION_ENCODER_DECODER_START_DOCSTRING = r""" - This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model - as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via - [`~AutoModel.from_pretrained`] function and the decoder is loaded via [`~AutoModelForCausalLM.from_pretrained`] - function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream - generative task, like image captioning. - - The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation - tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation - Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi - Zhou, Wei Li, Peter J. Liu. - - Additionally, in [TrOCR: Transformer-based Optical Character Recognition with Pre-trained - Models](https://arxiv.org/abs/2109.10282) it is shown how leveraging large pretrained vision models for optical - character recognition (OCR) yields a significant performance improvement. - - After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any - other models (see the examples for more information). - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`VisionEncoderDecoderConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VISION_ENCODER_DECODER_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using an image processor (e.g. if you use ViT as the encoder, - you should use [`AutoImageProcessor`]). See [`ViTImageProcessor.__call__`] for details. - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - - For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the - right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`. - decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - encoder_outputs (`tuple(torch.FloatTensor)`, *optional*): - This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor - of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the - decoder. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. This is useful if you want more control over how to convert `decoder_input_ids` indices - into associated vectors than the model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, - ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - If set to `True`, the model will return a [`~utils.Seq2SeqLMOutput`] instead of a plain tuple. - kwargs (*optional*): Remaining dictionary of keyword arguments. Keyword arguments come in two flavors: - - - Without a prefix which will be input as `**encoder_kwargs` for the encoder forward function. - - With a *decoder_* prefix which will be input as `**decoder_kwargs` for the decoder forward function. -""" - - -@add_start_docstrings(VISION_ENCODER_DECODER_START_DOCSTRING) +@auto_docstring class VisionEncoderDecoderModel(PreTrainedModel, GenerationMixin): r""" [`VisionEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with @@ -169,6 +77,12 @@ class VisionEncoderDecoderModel(PreTrainedModel, GenerationMixin): encoder: Optional[PreTrainedModel] = None, decoder: Optional[PreTrainedModel] = None, ): + r""" + encoder (`PreTrainedModel`, *optional*): + The encoder model to use. + decoder (`PreTrainedModel`, *optional*): + The decoder model to use. + """ if config is None and (encoder is None or decoder is None): raise ValueError("Either a configuration or an encoder and a decoder has to be provided.") if config is None: @@ -521,8 +435,7 @@ class VisionEncoderDecoderModel(PreTrainedModel, GenerationMixin): config.tie_word_embeddings = False return cls(encoder=encoder, decoder=decoder, config=config) - @add_start_docstrings_to_model_forward(VISION_ENCODER_DECODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -539,7 +452,30 @@ class VisionEncoderDecoderModel(PreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: r""" - Returns: + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the + right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`. + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert `decoder_input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, + ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Examples: diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 7b2e262e87..f7c3e1816f 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -20,7 +20,7 @@ import torch from torch import nn from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from ..auto.configuration_auto import AutoConfig from ..auto.modeling_auto import AutoModel from ..clip.modeling_clip import CLIPOutput, CLIPVisionConfig, CLIPVisionModel @@ -29,121 +29,6 @@ from .configuration_vision_text_dual_encoder import VisionTextDualEncoderConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "VisionTextDualEncoderConfig" - -VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r""" - This class can be used to initialize a vision-text dual encoder model with any pretrained vision autoencoding model - as the vision encoder and any pretrained text model as the text encoder. The vision and text encoders are loaded - via the [`~AutoModel.from_pretrained`] method. The projection layers are automatically added to the model and - should be fine-tuned on a downstream task, like contrastive image-text modeling. - - In [LiT: Zero-Shot Transfer with Locked-image Text Tuning](https://arxiv.org/abs/2111.07991) it is shown how - leveraging pre-trained (locked/frozen) image and text model for contrastive learning yields significant improvement - on new zero-shot vision tasks such as image classification or retrieval. - - After such a Vision-Text-Dual-Encoder model has been trained/fine-tuned, it can be saved/loaded just like any other - models (see the examples for more information). - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`VisionEncoderDecoderConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -VISION_TEXT_DUAL_ENCODER_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -VISION_TEXT_DUAL_ENCODER_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - an image processor (e.g. if you use ViT as the encoder, you should use [`AutoImageProcessor`]). See - [`ViTImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # Copied from transformers.models.clip.modeling_clip.contrastive_loss def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: @@ -157,7 +42,7 @@ def clip_loss(similarity: torch.Tensor) -> torch.Tensor: return (caption_loss + image_loss) / 2.0 -@add_start_docstrings(VISION_TEXT_DUAL_ENCODER_START_DOCSTRING) +@auto_docstring class VisionTextDualEncoderModel(PreTrainedModel): config_class = VisionTextDualEncoderConfig base_model_prefix = "vision_text_dual_encoder" @@ -170,6 +55,12 @@ class VisionTextDualEncoderModel(PreTrainedModel): vision_model: Optional[PreTrainedModel] = None, text_model: Optional[PreTrainedModel] = None, ): + r""" + vision_model (`PreTrainedModel`): + The vision model to use. + text_model (`PreTrainedModel`): + The text model to use. + """ if config is None and (vision_model is None or text_model is None): raise ValueError("Either a configuration or an vision and a text model has to be provided") @@ -209,7 +100,7 @@ class VisionTextDualEncoderModel(PreTrainedModel): self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False) self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value)) - @add_start_docstrings_to_model_forward(VISION_TEXT_DUAL_ENCODER_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids=None, @@ -251,7 +142,7 @@ class VisionTextDualEncoderModel(PreTrainedModel): return text_features - @add_start_docstrings_to_model_forward(VISION_TEXT_DUAL_ENCODER_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_image_features( self, pixel_values=None, @@ -293,8 +184,7 @@ class VisionTextDualEncoderModel(PreTrainedModel): return image_features - @add_start_docstrings_to_model_forward(VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CLIPOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -308,7 +198,8 @@ class VisionTextDualEncoderModel(PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], CLIPOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: diff --git a/src/transformers/models/visual_bert/modeling_visual_bert.py b/src/transformers/models/visual_bert/modeling_visual_bert.py index 2d81258a89..60f237b3b3 100755 --- a/src/transformers/models/visual_bert/modeling_visual_bert.py +++ b/src/transformers/models/visual_bert/modeling_visual_bert.py @@ -32,21 +32,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_visual_bert import VisualBertConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "VisualBertConfig" -_CHECKPOINT_FOR_DOC = "uclanlp/visualbert-vqa-coco-pre" - class VisualBertEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings and visual embeddings.""" @@ -507,12 +498,8 @@ class VisualBertPreTrainingHeads(nn.Module): return prediction_scores, seq_relationship_score +@auto_docstring class VisualBertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VisualBertConfig base_model_prefix = "visual_bert" supports_gradient_checkpointing = True @@ -566,104 +553,19 @@ class VisualBertForPreTrainingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -VISUAL_BERT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`VisualBertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VISUAL_BERT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - - visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): - The embedded representation of the visual inputs, generally derived using using an object detector. - - visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): - Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): - Segment token indices to indicate different portions of the visual embeds. - - [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the - *visual_token_type_ids* to *1* for all tokens. - - image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): - Image-Text alignment uses to decide the position IDs of the visual embeddings. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare VisualBert Model transformer outputting raw hidden-states without any specific head on top.", - VISUAL_BERT_START_DOCSTRING, -) -class VisualBertModel(VisualBertPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. """ - +) +class VisualBertModel(VisualBertPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -694,8 +596,7 @@ class VisualBertModel(VisualBertPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VISUAL_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -713,8 +614,22 @@ class VisualBertModel(VisualBertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPooling]: r""" + visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): + The embedded representation of the visual inputs, generally derived using using an object detector. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: - Returns: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Segment token indices to indicate different portions of the visual embeds. + + [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the + *visual_token_type_ids* to *1* for all tokens. + image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): + Image-Text alignment uses to decide the position IDs of the visual embeddings. Example: @@ -845,12 +760,11 @@ class VisualBertModel(VisualBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" VisualBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `sentence-image prediction (classification)` head. - """, - VISUAL_BERT_START_DOCSTRING, + """ ) class VisualBertForPreTraining(VisualBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -871,8 +785,7 @@ class VisualBertForPreTraining(VisualBertPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(VISUAL_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=VisualBertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -892,6 +805,22 @@ class VisualBertForPreTraining(VisualBertPreTrainedModel): sentence_image_labels: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], VisualBertForPreTrainingOutput]: r""" + visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): + The embedded representation of the visual inputs, generally derived using using an object detector. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Segment token indices to indicate different portions of the visual embeds. + + [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the + *visual_token_type_ids* to *1* for all tokens. + image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): + Image-Text alignment uses to decide the position IDs of the visual embeddings. labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the @@ -903,8 +832,6 @@ class VisualBertForPreTraining(VisualBertPreTrainedModel): - 0 indicates sequence B is a matching pair of sequence A for the given image, - 1 indicates sequence B is a random sequence w.r.t A for the given image. - Returns: - Example: ```python @@ -991,13 +918,7 @@ class VisualBertForPreTraining(VisualBertPreTrainedModel): ) -@add_start_docstrings( - """ - VisualBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and - a softmax) e.g. for VCR tasks. - """, - VISUAL_BERT_START_DOCSTRING, -) +@auto_docstring class VisualBertForMultipleChoice(VisualBertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1009,10 +930,7 @@ class VisualBertForMultipleChoice(VisualBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - VISUAL_BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @replace_return_docstrings(output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1031,13 +949,51 @@ class VisualBertForMultipleChoice(VisualBertPreTrainedModel): labels: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): + The embedded representation of the visual inputs, generally derived using using an object detector. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Segment token indices to indicate different portions of the visual embeds. + + [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the + *visual_token_type_ids* to *1* for all tokens. + image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): + Image-Text alignment uses to decide the position IDs of the visual embeddings. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) - Returns: - Example: ```python @@ -1144,12 +1100,11 @@ class VisualBertForMultipleChoice(VisualBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" VisualBert Model with a classification/regression head on top (a dropout and a linear layer on top of the pooled output) for VQA. - """, - VISUAL_BERT_START_DOCSTRING, + """ ) class VisualBertForQuestionAnswering(VisualBertPreTrainedModel): def __init__(self, config): @@ -1163,8 +1118,7 @@ class VisualBertForQuestionAnswering(VisualBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VISUAL_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1183,12 +1137,26 @@ class VisualBertForQuestionAnswering(VisualBertPreTrainedModel): labels: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): + The embedded representation of the visual inputs, generally derived using using an object detector. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Segment token indices to indicate different portions of the visual embeds. + + [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the + *visual_token_type_ids* to *1* for all tokens. + image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): + Image-Text alignment uses to decide the position IDs of the visual embeddings. labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. A KLDivLoss is computed between the labels and the returned logits. - Returns: - Example: ```python @@ -1270,12 +1238,11 @@ class VisualBertForQuestionAnswering(VisualBertPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" VisualBert Model with a sequence classification head on top (a dropout and a linear layer on top of the pooled output) for Visual Reasoning e.g. for NLVR task. - """, - VISUAL_BERT_START_DOCSTRING, + """ ) class VisualBertForVisualReasoning(VisualBertPreTrainedModel): def __init__(self, config): @@ -1289,8 +1256,7 @@ class VisualBertForVisualReasoning(VisualBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VISUAL_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1309,12 +1275,26 @@ class VisualBertForVisualReasoning(VisualBertPreTrainedModel): labels: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): + The embedded representation of the visual inputs, generally derived using using an object detector. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Segment token indices to indicate different portions of the visual embeds. + + [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the + *visual_token_type_ids* to *1* for all tokens. + image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): + Image-Text alignment uses to decide the position IDs of the visual embeddings. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. A classification loss is computed (Cross-Entropy) against these labels. - Returns: - Example: ```python @@ -1430,12 +1410,11 @@ class VisualBertRegionToPhraseAttention(nn.Module): return attention_scores -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" VisualBert Model with a Masked Language Modeling head and an attention layer on top for Region-to-Phrase Alignment e.g. for Flickr30 Entities task. - """, - VISUAL_BERT_START_DOCSTRING, + """ ) class VisualBertForRegionToPhraseAlignment(VisualBertPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.bias"] @@ -1451,8 +1430,7 @@ class VisualBertForRegionToPhraseAlignment(VisualBertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VISUAL_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1472,15 +1450,28 @@ class VisualBertForRegionToPhraseAlignment(VisualBertPreTrainedModel): labels: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*): + The embedded representation of the visual inputs, generally derived using using an object detector. + visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Mask to avoid performing attention on visual embeddings. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + visual_token_type_ids (`torch.LongTensor` of shape `(batch_size, visual_seq_length)`, *optional*): + Segment token indices to indicate different portions of the visual embeds. + + [What are token type IDs?](../glossary#token-type-ids) The authors of VisualBERT set the + *visual_token_type_ids* to *1* for all tokens. + image_text_alignment (`torch.LongTensor` of shape `(batch_size, visual_seq_length, alignment_number)`, *optional*): + Image-Text alignment uses to decide the position IDs of the visual embeddings. region_to_phrase_position (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*): The positions depicting the position of the image embedding corresponding to the textual tokens. - labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length, visual_sequence_length)`, *optional*): Labels for computing the masked language modeling loss. KLDivLoss is computed against these labels and the outputs from the attention layer. - Returns: - Example: ```python diff --git a/src/transformers/models/vit/image_processing_vit_fast.py b/src/transformers/models/vit/image_processing_vit_fast.py index 61277792cd..20aa73f73c 100644 --- a/src/transformers/models/vit/image_processing_vit_fast.py +++ b/src/transformers/models/vit/image_processing_vit_fast.py @@ -15,7 +15,6 @@ """Fast Image processor class for ViT.""" from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, BaseImageProcessorFast, ) from ...image_utils import ( @@ -24,14 +23,11 @@ from ...image_utils import ( PILImageResampling, ) from ...utils import ( - add_start_docstrings, + auto_docstring, ) -@add_start_docstrings( - "Constructs a fast ViT image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, -) +@auto_docstring class ViTImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_STANDARD_MEAN diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index d757aeaf28..6cae97942f 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -32,30 +32,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_vit import ViTConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "ViTConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "google/vit-base-patch16-224-in21k" -_EXPECTED_OUTPUT_SHAPE = [1, 197, 768] - -# Image classification docstring -_IMAGE_CLASS_CHECKPOINT = "google/vit-base-patch16-224" -_IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" - class ViTEmbeddings(nn.Module): """ @@ -457,12 +439,8 @@ class ViTEncoder(nn.Module): ) +@auto_docstring class ViTPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ViTConfig base_model_prefix = "vit" main_input_name = "pixel_values" @@ -501,48 +479,15 @@ class ViTPreTrainedModel(PreTrainedModel): module.mask_token.data.zero_() -VIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ViTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ViT Model transformer outputting raw hidden-states without any specific head on top.", - VIT_START_DOCSTRING, -) +@auto_docstring class ViTModel(ViTPreTrainedModel): def __init__(self, config: ViTConfig, add_pooling_layer: bool = True, use_mask_token: bool = False): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config @@ -566,14 +511,7 @@ class ViTModel(ViTPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -651,8 +589,9 @@ class ViTPooler(nn.Module): return pooled_output -@add_start_docstrings( - """ViT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886). +@auto_docstring( + custom_intro=""" + ViT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886). @@ -660,8 +599,7 @@ class ViTPooler(nn.Module): directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - """, - VIT_START_DOCSTRING, + """ ) class ViTForMaskedImageModeling(ViTPreTrainedModel): def __init__(self, config: ViTConfig) -> None: @@ -681,8 +619,7 @@ class ViTForMaskedImageModeling(ViTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -697,8 +634,6 @@ class ViTForMaskedImageModeling(ViTPreTrainedModel): bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, ViTForMaskedImageModeling @@ -777,8 +712,8 @@ class ViTForMaskedImageModeling(ViTPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ViT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. @@ -789,8 +724,7 @@ class ViTForMaskedImageModeling(ViTPreTrainedModel): position embeddings to the higher resolution. - """, - VIT_START_DOCSTRING, + """ ) class ViTForImageClassification(ViTPreTrainedModel): def __init__(self, config: ViTConfig) -> None: @@ -805,13 +739,7 @@ class ViTForImageClassification(ViTPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, diff --git a/src/transformers/models/vit_mae/modeling_vit_mae.py b/src/transformers/models/vit_mae/modeling_vit_mae.py index 4636519ee6..541015c7da 100755 --- a/src/transformers/models/vit_mae/modeling_vit_mae.py +++ b/src/transformers/models/vit_mae/modeling_vit_mae.py @@ -28,22 +28,12 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import ModelOutput, auto_docstring, logging, torch_int from .configuration_vit_mae import ViTMAEConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ViTMAEConfig" -_CHECKPOINT_FOR_DOC = "facebook/vit-mae-base" - @dataclass class ViTMAEModelOutput(ModelOutput): @@ -635,12 +625,8 @@ class ViTMAEEncoder(nn.Module): ) +@auto_docstring class ViTMAEPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ViTMAEConfig base_model_prefix = "vit" main_input_name = "pixel_values" @@ -666,47 +652,7 @@ class ViTMAEPreTrainedModel(PreTrainedModel): module.decoder_pos_embed.data.zero_() -VIT_MAE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ViTMAEConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VIT_MAE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - interpolate_pos_encoding (`bool`, *optional*, default `False`): - Whether to interpolate the pre-trained position encodings. This is mainly used to use the model on higher - resolution images. -""" - - -@add_start_docstrings( - "The bare ViTMAE Model transformer outputting raw hidden-states without any specific head on top.", - VIT_MAE_START_DOCSTRING, -) +@auto_docstring class ViTMAEModel(ViTMAEPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -731,8 +677,7 @@ class ViTMAEModel(ViTMAEPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VIT_MAE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ViTMAEModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -744,7 +689,11 @@ class ViTMAEModel(ViTMAEPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> Union[Tuple, ViTMAEModelOutput]: r""" - Returns: + interpolate_pos_encoding (`bool`, *optional*, default `False`): + Whether to interpolate the pre-trained position encodings. This is mainly used to use the model on higher + resolution images. + noise (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mainly used for testing purposes to control randomness and maintain the reproducibility Examples: @@ -948,8 +897,9 @@ class ViTMAEDecoder(nn.Module): ) -@add_start_docstrings( - """The ViTMAE Model transformer with the decoder on top for self-supervised pre-training. +@auto_docstring( + custom_intro=""" + The ViTMAE Model transformer with the decoder on top for self-supervised pre-training. @@ -957,9 +907,7 @@ class ViTMAEDecoder(nn.Module): directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). - - """, - VIT_MAE_START_DOCSTRING, + """ ) class ViTMAEForPreTraining(ViTMAEPreTrainedModel): def __init__(self, config): @@ -1091,8 +1039,7 @@ class ViTMAEForPreTraining(ViTMAEPreTrainedModel): loss = (loss * mask).sum() / mask.sum() # mean loss on removed patches return loss - @add_start_docstrings_to_model_forward(VIT_MAE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ViTMAEForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1104,7 +1051,11 @@ class ViTMAEForPreTraining(ViTMAEPreTrainedModel): interpolate_pos_encoding: bool = False, ) -> Union[Tuple, ViTMAEForPreTrainingOutput]: r""" - Returns: + interpolate_pos_encoding (`bool`, *optional*, default `False`): + Whether to interpolate the pre-trained position encodings. This is mainly used to use the model on higher + resolution images. + noise (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mainly used for testing purposes to control randomness and maintain the reproducibility Examples: diff --git a/src/transformers/models/vit_msn/modeling_vit_msn.py b/src/transformers/models/vit_msn/modeling_vit_msn.py index fb5a3d56ba..16d61b9532 100644 --- a/src/transformers/models/vit_msn/modeling_vit_msn.py +++ b/src/transformers/models/vit_msn/modeling_vit_msn.py @@ -26,23 +26,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_vit_msn import ViTMSNConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ViTMSNConfig" -_CHECKPOINT_FOR_DOC = "facebook/vit-msn-small" - - class ViTMSNEmbeddings(nn.Module): """ Construct the CLS token, position and patch embeddings. Optionally, also the mask token. @@ -453,12 +443,8 @@ class ViTMSNEncoder(nn.Module): ) +@auto_docstring class ViTMSNPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ViTMSNConfig base_model_prefix = "vit" main_input_name = "pixel_values" @@ -487,48 +473,13 @@ class ViTMSNPreTrainedModel(PreTrainedModel): module.mask_token.data.zero_() -VIT_MSN_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ViTMSNConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VIT_MSN_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ViTMSN Model outputting raw hidden-states without any specific head on top.", - VIT_MSN_START_DOCSTRING, -) +@auto_docstring class ViTMSNModel(ViTMSNPreTrainedModel): def __init__(self, config: ViTMSNConfig, use_mask_token: bool = False): + r""" + use_mask_token (`bool`, *optional*, defaults to `False`): + Whether to use a mask token for masked image modeling. + """ super().__init__(config) self.config = config @@ -551,8 +502,7 @@ class ViTMSNModel(ViTMSNPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VIT_MSN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -567,8 +517,6 @@ class ViTMSNModel(ViTMSNPreTrainedModel): bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - Returns: - Examples: ```python @@ -630,12 +578,7 @@ class ViTMSNModel(ViTMSNPreTrainedModel): # Caution: We don't have the weights for the classification head yet. This class # is here for the users that are interested to fine-tune the base model (ViTMSNModel). -@add_start_docstrings( - """ - ViTMSN Model with an image classification head on top e.g. for ImageNet. - """, - VIT_MSN_START_DOCSTRING, -) +@auto_docstring class ViTMSNForImageClassification(ViTMSNPreTrainedModel): def __init__(self, config: ViTMSNConfig) -> None: super().__init__(config) @@ -649,8 +592,7 @@ class ViTMSNForImageClassification(ViTMSNPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VIT_MSN_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -662,8 +604,6 @@ class ViTMSNForImageClassification(ViTMSNPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[tuple, ImageClassifierOutput]: r""" - Returns: - Examples: ```python diff --git a/src/transformers/models/vitdet/modeling_vitdet.py b/src/transformers/models/vitdet/modeling_vitdet.py index 3d74052288..6b31137cf0 100644 --- a/src/transformers/models/vitdet/modeling_vitdet.py +++ b/src/transformers/models/vitdet/modeling_vitdet.py @@ -25,21 +25,13 @@ from torch import nn from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput, BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_vitdet import VitDetConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "VitDetConfig" - class VitDetEmbeddings(nn.Module): """ @@ -609,12 +601,8 @@ def caffe2_msra_fill(module: nn.Module) -> None: nn.init.constant_(module.bias, 0) +@auto_docstring class VitDetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VitDetConfig base_model_prefix = "vitdet" main_input_name = "pixel_values" @@ -665,44 +653,7 @@ class VitDetPreTrainedModel(PreTrainedModel): module.norm3.bias.data.zero_() -VITDET_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`VitDetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VITDET_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] - for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare VitDet Transformer model outputting raw hidden-states without any specific head on top.", - VITDET_START_DOCSTRING, -) +@auto_docstring class VitDetModel(VitDetPreTrainedModel): def __init__(self, config: VitDetConfig): super().__init__(config) @@ -725,8 +676,7 @@ class VitDetModel(VitDetPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VITDET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -735,9 +685,7 @@ class VitDetModel(VitDetPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: - """ - Returns: - + r""" Examples: ```python @@ -793,11 +741,10 @@ class VitDetModel(VitDetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ViTDet backbone, to be used with frameworks like Mask R-CNN. - """, - VITDET_START_DOCSTRING, + """ ) class VitDetBackbone(VitDetPreTrainedModel, BackboneMixin): def __init__(self, config): @@ -814,8 +761,7 @@ class VitDetBackbone(VitDetPreTrainedModel, BackboneMixin): def get_input_embeddings(self) -> VitDetEmbeddings: return self.embeddings.projection - @add_start_docstrings_to_model_forward(VITDET_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -823,9 +769,7 @@ class VitDetBackbone(VitDetPreTrainedModel, BackboneMixin): output_attentions: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> BackboneOutput: - """ - Returns: - + r""" Examples: ```python diff --git a/src/transformers/models/vitmatte/image_processing_vitmatte_fast.py b/src/transformers/models/vitmatte/image_processing_vitmatte_fast.py index 4355a02664..59f8a38335 100644 --- a/src/transformers/models/vitmatte/image_processing_vitmatte_fast.py +++ b/src/transformers/models/vitmatte/image_processing_vitmatte_fast.py @@ -19,8 +19,6 @@ from typing import Optional, Union from ...image_processing_utils import BatchFeature from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, group_images_by_shape, @@ -38,7 +36,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, filter_out_non_signature_kwargs, is_torch_available, is_torchvision_available, @@ -61,21 +59,19 @@ logger = logging.get_logger(__name__) class VitMatteFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + """ + do_pad (`bool`, *optional*, defaults to `True`): + Whether to pad the image to make the width and height divisible by `size_divisibility`. Can be overridden + by the `do_pad` parameter in the `preprocess` method. + size_divisibility (`int`, *optional*, defaults to 32): + The width and height of the image will be padded to be divisible by this number. + """ + do_pad: Optional[bool] size_divisibility: int -@add_start_docstrings( - "Constructs a fast VitMatte image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - do_pad (`bool`, *optional*, defaults to `True`): - Whether to pad the image to make the width and height divisible by `size_divisibility`. Can be overridden - by the `do_pad` parameter in the `preprocess` method. - size_divisibility (`int`, *optional*, defaults to 32): - The width and height of the image will be padded to be divisible by this number. - """, -) +@auto_docstring class VitMatteImageProcessorFast(BaseImageProcessorFast): do_rescale: bool = True rescale_factor: Union[int, float] = 1 / 255 @@ -89,22 +85,17 @@ class VitMatteImageProcessorFast(BaseImageProcessorFast): def __init__(self, **kwargs: Unpack[VitMatteFastImageProcessorKwargs]) -> None: super().__init__(**kwargs) - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ - do_pad (`bool`, *optional*, defaults to `True`): - Whether to pad the image to make the width and height divisible by `size_divisibility`. Can be overridden - by the `do_pad` parameter in the `preprocess` method. - size_divisibility (`int`, *optional*, defaults to 32): - The width and height of the image will be padded to be divisible by this number. - """, - ) + @auto_docstring def preprocess( self, images: list["torch.Tensor"], trimaps: list["torch.Tensor"], **kwargs: Unpack[VitMatteFastImageProcessorKwargs], ) -> BatchFeature: + r""" + trimaps (`list[torch.Tensor]`): + The trimaps to preprocess. + """ validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys()) # Set default kwargs from self. This ensures that if a kwarg is not provided # by the user, it gets its default value from the instance, or is set to None. diff --git a/src/transformers/models/vitmatte/modeling_vitmatte.py b/src/transformers/models/vitmatte/modeling_vitmatte.py index aa3aa5b883..9e09428586 100644 --- a/src/transformers/models/vitmatte/modeling_vitmatte.py +++ b/src/transformers/models/vitmatte/modeling_vitmatte.py @@ -21,20 +21,11 @@ import torch from torch import nn from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring from ...utils.backbone_utils import load_backbone from .configuration_vitmatte import VitMatteConfig -# General docstring -_CONFIG_FOR_DOC = "VitMatteConfig" - - @dataclass class ImageMattingOutput(ModelOutput): """ @@ -63,12 +54,8 @@ class ImageMattingOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@auto_docstring class VitMattePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VitMatteConfig main_input_name = "pixel_values" supports_gradient_checkpointing = True @@ -224,35 +211,10 @@ class VitMatteDetailCaptureModule(nn.Module): return alphas -VITMATTE_START_DOCSTRING = r""" - Parameters: - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - config ([`UperNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VITMATTE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`VitMatteImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers in case the backbone has them. See - `attentions` under returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers of the backbone. See `hidden_states` under - returned tensors for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """ViTMatte framework leveraging any vision backbone e.g. for ADE20k, CityScapes.""", - VITMATTE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + ViTMatte framework leveraging any vision backbone e.g. for ADE20k, CityScapes. + """ ) class VitMatteForImageMatting(VitMattePreTrainedModel): def __init__(self, config): @@ -265,8 +227,7 @@ class VitMatteForImageMatting(VitMattePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VITMATTE_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=ImageMattingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -275,12 +236,10 @@ class VitMatteForImageMatting(VitMattePreTrainedModel): labels: Optional[torch.Tensor] = None, return_dict: Optional[bool] = None, ): - """ + r""" labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth image matting for computing the loss. - Returns: - Examples: ```python diff --git a/src/transformers/models/vitpose/modeling_vitpose.py b/src/transformers/models/vitpose/modeling_vitpose.py index dfe9738abf..d872bd75db 100644 --- a/src/transformers/models/vitpose/modeling_vitpose.py +++ b/src/transformers/models/vitpose/modeling_vitpose.py @@ -24,10 +24,8 @@ from torch import nn from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, ) from ...utils.backbone_utils import load_backbone from .configuration_vitpose import VitPoseConfig @@ -36,7 +34,6 @@ from .configuration_vitpose import VitPoseConfig logger = logging.get_logger(__name__) # General docstring -_CONFIG_FOR_DOC = "VitPoseConfig" @dataclass @@ -67,12 +64,8 @@ class VitPoseEstimatorOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor, ...]] = None +@auto_docstring class VitPosePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VitPoseConfig base_model_prefix = "vit" main_input_name = "pixel_values" @@ -93,42 +86,6 @@ class VitPosePreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -VITPOSE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`VitPoseConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VITPOSE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`VitPoseImageProcessor`]. See - [`VitPoseImageProcessor.__call__`] for details. - - dataset_index (`torch.Tensor` of shape `(batch_size,)`): - Index to use in the Mixture-of-Experts (MoE) blocks of the backbone. - - This corresponds to the dataset index used during training, e.g. For the single dataset index 0 refers to the corresponding dataset. For the multiple datasets index 0 refers to dataset A (e.g. MPII) and index 1 refers to dataset B (e.g. CrowdPose). - - flip_pairs (`torch.tensor`, *optional*): - Whether to mirror pairs of keypoints (for example, left ear -- right ear). - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - def flip_back(output_flipped, flip_pairs, target_type="gaussian-heatmap"): """Flip the flipped heatmaps back to the original form. @@ -234,9 +191,10 @@ class VitPoseClassicDecoder(nn.Module): return heatmaps -@add_start_docstrings( - "The VitPose model with a pose estimation head on top.", - VITPOSE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VitPose model with a pose estimation head on top. + """ ) class VitPoseForPoseEstimation(VitPosePreTrainedModel): def __init__(self, config: VitPoseConfig) -> None: @@ -257,8 +215,7 @@ class VitPoseForPoseEstimation(VitPosePreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VITPOSE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=VitPoseEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -269,8 +226,13 @@ class VitPoseForPoseEstimation(VitPosePreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[tuple, VitPoseEstimatorOutput]: - """ - Returns: + r""" + dataset_index (`torch.Tensor` of shape `(batch_size,)`): + Index to use in the Mixture-of-Experts (MoE) blocks of the backbone. + + This corresponds to the dataset index used during training, e.g. For the single dataset index 0 refers to the corresponding dataset. For the multiple datasets index 0 refers to dataset A (e.g. MPII) and index 1 refers to dataset B (e.g. CrowdPose). + flip_pairs (`torch.tensor`, *optional*): + Whether to mirror pairs of keypoints (for example, left ear -- right ear). Examples: diff --git a/src/transformers/models/vitpose_backbone/modeling_vitpose_backbone.py b/src/transformers/models/vitpose_backbone/modeling_vitpose_backbone.py index 8abce8dfdc..8f329de6e1 100644 --- a/src/transformers/models/vitpose_backbone/modeling_vitpose_backbone.py +++ b/src/transformers/models/vitpose_backbone/modeling_vitpose_backbone.py @@ -30,21 +30,13 @@ from ...activations import ACT2FN from ...modeling_outputs import BackboneOutput, BaseModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.backbone_utils import BackboneMixin from .configuration_vitpose_backbone import VitPoseBackboneConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "VitPoseBackboneConfig" - class VitPoseBackbonePatchEmbeddings(nn.Module): """Image to Patch Embedding.""" @@ -413,12 +405,8 @@ class VitPoseBackboneEncoder(nn.Module): ) +@auto_docstring class VitPoseBackbonePreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VitPoseBackboneConfig base_model_prefix = "vit" main_input_name = "pixel_values" @@ -448,47 +436,10 @@ class VitPoseBackbonePreTrainedModel(PreTrainedModel): ).to(module.position_embeddings.dtype) -VITPOSE_BACKBONE_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`VitPoseBackboneConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VITPOSE_BACKBONE_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. - - dataset_index (`torch.Tensor` of shape `(batch_size,)`): - Index to use in the Mixture-of-Experts (MoE) blocks of the backbone. - - This corresponds to the dataset index used during training, e.g. index 0 refers to COCO. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The VitPose backbone useful for downstream tasks.", - VITPOSE_BACKBONE_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The VitPose backbone useful for downstream tasks. + """ ) class VitPoseBackbone(VitPoseBackbonePreTrainedModel, BackboneMixin): def __init__(self, config: VitPoseBackboneConfig): @@ -504,8 +455,7 @@ class VitPoseBackbone(VitPoseBackbonePreTrainedModel, BackboneMixin): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VITPOSE_BACKBONE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.Tensor, @@ -515,8 +465,11 @@ class VitPoseBackbone(VitPoseBackbonePreTrainedModel, BackboneMixin): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ): - """ - Returns: + r""" + dataset_index (`torch.Tensor` of shape `(batch_size,)`): + Index to use in the Mixture-of-Experts (MoE) blocks of the backbone. + + This corresponds to the dataset index used during training, e.g. index 0 refers to COCO. Examples: diff --git a/src/transformers/models/vits/modeling_vits.py b/src/transformers/models/vits/modeling_vits.py index 59483d3e61..3acca4036b 100644 --- a/src/transformers/models/vits/modeling_vits.py +++ b/src/transformers/models/vits/modeling_vits.py @@ -27,22 +27,15 @@ from ...activations import ACT2FN from ...integrations.deepspeed import is_deepspeed_zero3_enabled from ...integrations.fsdp import is_fsdp_managed_module from ...modeling_attn_mask_utils import _prepare_4d_attention_mask -from ...modeling_outputs import ( - BaseModelOutput, - ModelOutput, -) +from ...modeling_outputs import BaseModelOutput, ModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import auto_docstring, logging from .configuration_vits import VitsConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "VitsConfig" - - @dataclass class VitsModelOutput(ModelOutput): """ @@ -1246,12 +1239,8 @@ class VitsTextEncoder(nn.Module): ) +@auto_docstring class VitsPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VitsConfig base_model_prefix = "vits" main_input_name = "input_ids" @@ -1277,57 +1266,10 @@ class VitsPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -VITS_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`VitsConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -VITS_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - speaker_id (`int`, *optional*): - Which speaker embedding to use. Only used for multispeaker models. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The complete VITS model, for text-to-speech synthesis.", - VITS_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The complete VITS model, for text-to-speech synthesis. + """ ) class VitsModel(VitsPreTrainedModel): def __init__(self, config: VitsConfig): @@ -1359,8 +1301,7 @@ class VitsModel(VitsPreTrainedModel): def get_encoder(self): return self.text_encoder - @add_start_docstrings_to_model_forward(VITS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=VitsModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1372,12 +1313,12 @@ class VitsModel(VitsPreTrainedModel): labels: Optional[torch.FloatTensor] = None, ) -> Union[Tuple[Any], VitsModelOutput]: r""" + speaker_id (`int`, *optional*): + Which speaker embedding to use. Only used for multispeaker models. labels (`torch.FloatTensor` of shape `(batch_size, config.spectrogram_bins, sequence_length)`, *optional*): Float values of target spectrogram. Timesteps set to `-100.0` are ignored (masked) for the loss computation. - Returns: - Example: ```python diff --git a/src/transformers/models/vivit/modeling_vivit.py b/src/transformers/models/vivit/modeling_vivit.py index 669106239a..15097bd69a 100755 --- a/src/transformers/models/vivit/modeling_vivit.py +++ b/src/transformers/models/vivit/modeling_vivit.py @@ -25,21 +25,12 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, - torch_int, -) +from ...utils import auto_docstring, logging, torch_int from .configuration_vivit import VivitConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "google/vivit-b-16x2-kinetics400" -_CONFIG_FOR_DOC = "VivitConfig" - class VivitTubeletEmbeddings(nn.Module): """ @@ -456,12 +447,8 @@ class VivitPooler(nn.Module): return pooled_output +@auto_docstring class VivitPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = VivitConfig base_model_prefix = "vivit" main_input_name = "pixel_values" @@ -490,48 +477,13 @@ class VivitPreTrainedModel(PreTrainedModel): module.position_embeddings.data.zero_() -VIVIT_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`VivitConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -VIVIT_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`VivitImageProcessor`]. See - [`VivitImageProcessor.preprocess`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare ViViT Transformer model outputting raw hidden-states without any specific head on top.", - VIVIT_START_DOCSTRING, -) +@auto_docstring class VivitModel(VivitPreTrainedModel): def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -558,8 +510,7 @@ class VivitModel(VivitPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(VIVIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -570,8 +521,6 @@ class VivitModel(VivitPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.FloatTensor], BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -680,20 +629,19 @@ class VivitModel(VivitPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + ViViT Transformer model with a video classification head on top (a linear layer on top of the final hidden state of the + [CLS] token) e.g. for Kinetics-400. + + + + Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by + setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained + position embeddings to the higher resolution. + + """ - ViViT Transformer model with a video classification head on top (a linear layer on top of the final hidden state of the -[CLS] token) e.g. for Kinetics-400. - - - - Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by - setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained - position embeddings to the higher resolution. - - - """, - VIVIT_START_DOCSTRING, ) class VivitForVideoClassification(VivitPreTrainedModel): def __init__(self, config): @@ -708,8 +656,7 @@ class VivitForVideoClassification(VivitPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(VIVIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -726,8 +673,6 @@ class VivitForVideoClassification(VivitPreTrainedModel): config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Examples: ```python diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py index f3708eaa2d..8b328fcd1a 100755 --- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py @@ -41,15 +41,12 @@ from ...modeling_outputs import ( from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, cached_file, check_torch_load_is_safe, is_peft_available, is_safetensors_available, logging, - replace_return_docstrings, ) from .configuration_wav2vec2 import Wav2Vec2Config @@ -70,30 +67,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 2 -# General docstring -_CONFIG_FOR_DOC = "Wav2Vec2Config" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/wav2vec2-base-960h" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 53.48 - -# Audio class docstring -_SEQ_CLASS_CHECKPOINT = "superb/wav2vec2-base-superb-ks" -_SEQ_CLASS_EXPECTED_OUTPUT = "'_unknown_'" -_SEQ_CLASS_EXPECTED_LOSS = 6.54 - -# Frame class docstring -_FRAME_CLASS_CHECKPOINT = "anton-l/wav2vec2-base-superb-sd" -_FRAME_EXPECTED_OUTPUT = [0, 0] - -# Speaker Verification docstring -_XVECTOR_CHECKPOINT = "anton-l/wav2vec2-base-superb-sv" -_XVECTOR_EXPECTED_OUTPUT = 0.98 - @dataclass class Wav2Vec2ForPreTrainingOutput(ModelOutput): @@ -1320,12 +1293,8 @@ class Wav2Vec2AttnAdapterLayer(nn.Module): return hidden_states +@auto_docstring class Wav2Vec2PreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Wav2Vec2Config base_model_prefix = "wav2vec2" main_input_name = "input_values" @@ -1639,67 +1608,7 @@ class Wav2Vec2PreTrainedModel(PreTrainedModel): self.target_lang = target_lang -WAV2VEC2_START_DOCSTRING = r""" - Wav2Vec2 was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech - Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael - Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Wav2Vec2Config`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -WAV2VEC2_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, such as - [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), `attention_mask` should **not** be - passed to avoid degraded performance when doing batched inference. For such models `input_values` should - simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly - different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare Wav2Vec2 Model transformer outputting raw hidden-states without any specific head on top.", - WAV2VEC2_START_DOCSTRING, -) +@auto_docstring class Wav2Vec2Model(Wav2Vec2PreTrainedModel): def __init__(self, config: Wav2Vec2Config): super().__init__(config) @@ -1786,14 +1695,7 @@ class Wav2Vec2Model(Wav2Vec2PreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Wav2Vec2BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1803,6 +1705,11 @@ class Wav2Vec2Model(Wav2Vec2PreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Wav2Vec2BaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1847,7 +1754,11 @@ class Wav2Vec2Model(Wav2Vec2PreTrainedModel): ) -@add_start_docstrings("""Wav2Vec2 Model with a quantizer and `VQ` head on top.""", WAV2VEC2_START_DOCSTRING) +@auto_docstring( + custom_intro=""" + Wav2Vec2 Model with a quantizer and `VQ` head on top. + """ +) class Wav2Vec2ForPreTraining(Wav2Vec2PreTrainedModel): def __init__(self, config: Wav2Vec2Config): super().__init__(config) @@ -1908,8 +1819,7 @@ class Wav2Vec2ForPreTraining(Wav2Vec2PreTrainedModel): logits = logits / temperature return logits - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Wav2Vec2ForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1928,8 +1838,6 @@ class Wav2Vec2ForPreTraining(Wav2Vec2PreTrainedModel): Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss. Required input for pre-training. - Returns: - Example: ```python @@ -2071,7 +1979,7 @@ class Wav2Vec2ForPreTraining(Wav2Vec2PreTrainedModel): ) -@add_start_docstrings("""Wav2Vec2 Model with a `language modeling` head on top.""", WAV2VEC2_START_DOCSTRING) +@auto_docstring class Wav2Vec2ForMaskedLM(Wav2Vec2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2087,7 +1995,7 @@ class Wav2Vec2ForMaskedLM(Wav2Vec2PreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_values: torch.FloatTensor, @@ -2117,18 +2025,19 @@ class Wav2Vec2ForMaskedLM(Wav2Vec2PreTrainedModel): return MaskedLMOutput(logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions) -@add_start_docstrings( - """Wav2Vec2 Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAV2VEC2_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Wav2Vec2 Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """ +) +class Wav2Vec2ForCTC(Wav2Vec2PreTrainedModel): + def __init__(self, config, target_lang: Optional[str] = None): + r""" target_lang (`str`, *optional*): Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or adapter..bin. Only relevant when using an instance of [`Wav2Vec2ForCTC`] with adapters. Uses 'eng' by default. - """, -) -class Wav2Vec2ForCTC(Wav2Vec2PreTrainedModel): - def __init__(self, config, target_lang: Optional[str] = None): + """ super().__init__(config) self.wav2vec2 = Wav2Vec2Model(config) @@ -2199,14 +2108,7 @@ class Wav2Vec2ForCTC(Wav2Vec2PreTrainedModel): for param in self.wav2vec2.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -2278,12 +2180,11 @@ class Wav2Vec2ForCTC(Wav2Vec2PreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Wav2Vec2 Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - WAV2VEC2_START_DOCSTRING, + """ ) class Wav2Vec2ForSequenceClassification(Wav2Vec2PreTrainedModel): def __init__(self, config): @@ -2330,15 +2231,7 @@ class Wav2Vec2ForSequenceClassification(Wav2Vec2PreTrainedModel): for param in self.wav2vec2.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_SEQ_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -2349,6 +2242,11 @@ class Wav2Vec2ForSequenceClassification(Wav2Vec2PreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -2402,12 +2300,7 @@ class Wav2Vec2ForSequenceClassification(Wav2Vec2PreTrainedModel): ) -@add_start_docstrings( - """ - Wav2Vec2 Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAV2VEC2_START_DOCSTRING, -) +@auto_docstring class Wav2Vec2ForAudioFrameClassification(Wav2Vec2PreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2452,14 +2345,7 @@ class Wav2Vec2ForAudioFrameClassification(Wav2Vec2PreTrainedModel): for param in self.wav2vec2.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_FRAME_CLASS_CHECKPOINT, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_FRAME_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -2470,6 +2356,11 @@ class Wav2Vec2ForAudioFrameClassification(Wav2Vec2PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -2569,11 +2460,10 @@ class TDNNLayer(nn.Module): return hidden_states -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Wav2Vec2 Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAV2VEC2_START_DOCSTRING, + """ ) class Wav2Vec2ForXVector(Wav2Vec2PreTrainedModel): def __init__(self, config): @@ -2637,14 +2527,7 @@ class Wav2Vec2ForXVector(Wav2Vec2PreTrainedModel): return input_lengths - @add_start_docstrings_to_model_forward(WAV2VEC2_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_XVECTOR_CHECKPOINT, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_XVECTOR_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -2655,6 +2538,11 @@ class Wav2Vec2ForXVector(Wav2Vec2PreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/wav2vec2_bert/modeling_wav2vec2_bert.py b/src/transformers/models/wav2vec2_bert/modeling_wav2vec2_bert.py index 27c809ac5a..ba606c0028 100644 --- a/src/transformers/models/wav2vec2_bert/modeling_wav2vec2_bert.py +++ b/src/transformers/models/wav2vec2_bert/modeling_wav2vec2_bert.py @@ -26,19 +26,10 @@ from ...modeling_outputs import ( XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_peft_available, -) +from ...utils import auto_docstring, is_peft_available from .configuration_wav2vec2_bert import Wav2Vec2BertConfig -# General docstring -_CONFIG_FOR_DOC = "Wav2Vec2BertConfig" - - class Wav2Vec2BertRotaryPositionalEmbedding(nn.Module): """Rotary positional embedding Reference : https://blog.eleuther.ai/rotary-embeddings/ Paper: https://arxiv.org/pdf/2104.09864.pdf @@ -723,12 +714,8 @@ class Wav2Vec2BertAdapterLayer(nn.Module): return hidden_states +@auto_docstring class Wav2Vec2BertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Wav2Vec2BertConfig base_model_prefix = "wav2vec2_bert" main_input_name = "input_features" @@ -935,61 +922,10 @@ def _compute_mask_indices( return spec_aug_mask -_PRETRAINED_CHECKPOINT_FOR_DOC = "hf-audio/wav2vec2-bert-CV16-en" -_EXPECTED_OUTPUT_SHAPE = [1, 146, 1024] - - -WAV2VEC2_BERT_START_DOCSTRING = r""" - Wav2Vec2Bert was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech - Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael - Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Wav2Vec2BertConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -WAV2VEC2_BERT_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - Wav2Vec2BertBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare Wav2Vec2Bert Model transformer outputting raw hidden-states without any specific head on top.", - WAV2VEC2_BERT_START_DOCSTRING, -) +@auto_docstring class Wav2Vec2BertModel(Wav2Vec2BertPreTrainedModel): def __init__(self, config: Wav2Vec2BertConfig): super().__init__(config) @@ -1057,14 +993,7 @@ class Wav2Vec2BertModel(Wav2Vec2BertPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_PRETRAINED_CHECKPOINT_FOR_DOC, - output_type=Wav2Vec2BertBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_features: Optional[torch.Tensor], @@ -1074,6 +1003,16 @@ class Wav2Vec2BertModel(Wav2Vec2BertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Wav2Vec2BertBaseModelOutput]: + r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1115,17 +1054,20 @@ class Wav2Vec2BertModel(Wav2Vec2BertPreTrainedModel): _HIDDEN_STATES_START_POSITION = 2 -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'mr quilter is the apostle of the middle classes and we are glad to welcome his gospel'" -_CTC_EXPECTED_LOSS = 17.04 - -@add_start_docstrings( - """Wav2Vec2Bert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAV2VEC2_BERT_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Wav2Vec2Bert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) class Wav2Vec2BertForCTC(Wav2Vec2BertPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.wav2vec2_bert = Wav2Vec2BertModel(config) @@ -1148,14 +1090,7 @@ class Wav2Vec2BertForCTC(Wav2Vec2BertPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_PRETRAINED_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_features: Optional[torch.Tensor], @@ -1166,6 +1101,11 @@ class Wav2Vec2BertForCTC(Wav2Vec2BertPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, CausalLMOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. @@ -1229,16 +1169,11 @@ class Wav2Vec2BertForCTC(Wav2Vec2BertPreTrainedModel): ) -# Base docstring -_BASE_CHECKPOINT_FOR_DOC = "facebook/w2v-bert-2.0" - - -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Wav2Vec2Bert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like + SUPERB Keyword Spotting. """ - Wav2Vec2Bert Model with a sequence classification head on top (a linear layer over the pooled output) for - tasks like SUPERB Keyword Spotting. - """, - WAV2VEC2_BERT_START_DOCSTRING, ) class Wav2Vec2BertForSequenceClassification(Wav2Vec2BertPreTrainedModel): def __init__(self, config): @@ -1266,13 +1201,7 @@ class Wav2Vec2BertForSequenceClassification(Wav2Vec2BertPreTrainedModel): for param in self.wav2vec2_bert.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_BASE_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_features: Optional[torch.Tensor], @@ -1283,6 +1212,11 @@ class Wav2Vec2BertForSequenceClassification(Wav2Vec2BertPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1336,12 +1270,7 @@ class Wav2Vec2BertForSequenceClassification(Wav2Vec2BertPreTrainedModel): ) -@add_start_docstrings( - """ - Wav2Vec2Bert Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAV2VEC2_BERT_START_DOCSTRING, -) +@auto_docstring class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2BertPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1367,13 +1296,7 @@ class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2BertPreTrainedModel): for param in self.wav2vec2_bert.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_BASE_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_features: Optional[torch.Tensor], @@ -1384,6 +1307,11 @@ class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2BertPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1483,11 +1411,10 @@ class TDNNLayer(nn.Module): return hidden_states -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Wav2Vec2Bert Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAV2VEC2_BERT_START_DOCSTRING, + """ ) class Wav2Vec2BertForXVector(Wav2Vec2BertPreTrainedModel): def __init__(self, config): @@ -1532,13 +1459,7 @@ class Wav2Vec2BertForXVector(Wav2Vec2BertPreTrainedModel): return input_lengths - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_BASE_CHECKPOINT_FOR_DOC, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_features: Optional[torch.Tensor], @@ -1549,6 +1470,11 @@ class Wav2Vec2BertForXVector(Wav2Vec2BertPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/wav2vec2_bert/modular_wav2vec2_bert.py b/src/transformers/models/wav2vec2_bert/modular_wav2vec2_bert.py index 8447bde849..1b4693e978 100644 --- a/src/transformers/models/wav2vec2_bert/modular_wav2vec2_bert.py +++ b/src/transformers/models/wav2vec2_bert/modular_wav2vec2_bert.py @@ -18,7 +18,7 @@ from ...modeling_outputs import ( XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from ..wav2vec2.modeling_wav2vec2 import Wav2Vec2FeedForward, Wav2Vec2ForSequenceClassification, Wav2Vec2Model from ..wav2vec2_conformer.modeling_wav2vec2_conformer import ( Wav2Vec2ConformerForAudioFrameClassification, @@ -36,18 +36,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 2 -# General docstring -_CONFIG_FOR_DOC = "Wav2Vec2BertConfig" - -# Base docstring -_BASE_CHECKPOINT_FOR_DOC = "facebook/w2v-bert-2.0" -_PRETRAINED_CHECKPOINT_FOR_DOC = "hf-audio/wav2vec2-bert-CV16-en" -_EXPECTED_OUTPUT_SHAPE = [1, 146, 1024] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'mr quilter is the apostle of the middle classes and we are glad to welcome his gospel'" -_CTC_EXPECTED_LOSS = 17.04 - # Copied from transformers.models.seamless_m4t_v2.modeling_seamless_m4t_v2._compute_new_attention_mask def _compute_new_attention_mask(hidden_states: torch.Tensor, seq_lens: torch.Tensor): @@ -598,12 +586,8 @@ class Wav2Vec2BertAdapterLayer(nn.Module): return hidden_states +@auto_docstring class Wav2Vec2BertPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Wav2Vec2BertConfig base_model_prefix = "wav2vec2_bert" main_input_name = "input_features" @@ -691,41 +675,9 @@ class Wav2Vec2BertPreTrainedModel(PreTrainedModel): return attention_mask -WAV2VEC2_BERT_START_DOCSTRING = None - -WAV2VEC2_BERT_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - Wav2Vec2BertBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare Wav2Vec2Bert Model transformer outputting raw hidden-states without any specific head on top.", - WAV2VEC2_BERT_START_DOCSTRING, -) class Wav2Vec2BertModel(Wav2Vec2Model, Wav2Vec2BertPreTrainedModel): def __init__(self, config: Wav2Vec2BertConfig): Wav2Vec2BertPreTrainedModel.__init__(config) @@ -753,14 +705,6 @@ class Wav2Vec2BertModel(Wav2Vec2Model, Wav2Vec2BertPreTrainedModel): def freeze_feature_encoder(self): raise AttributeError("Not needed for Wav2Vec2Bert") - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_PRETRAINED_CHECKPOINT_FOR_DOC, - output_type=Wav2Vec2BertBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) def forward( self, input_features: Optional[torch.Tensor], @@ -770,6 +714,16 @@ class Wav2Vec2BertModel(Wav2Vec2Model, Wav2Vec2BertPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Wav2Vec2BertBaseModelOutput]: + r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -809,25 +763,19 @@ class Wav2Vec2BertModel(Wav2Vec2Model, Wav2Vec2BertPreTrainedModel): ) -@add_start_docstrings( - """Wav2Vec2Bert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAV2VEC2_BERT_START_DOCSTRING, -) class Wav2Vec2BertForCTC(Wav2Vec2ConformerForCTC): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) def freeze_feature_encoder(self): raise AttributeError("Not needed for Wav2Vec2Bert") - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_PRETRAINED_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) def forward( self, input_features: Optional[torch.Tensor], @@ -838,6 +786,11 @@ class Wav2Vec2BertForCTC(Wav2Vec2ConformerForCTC): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, CausalLMOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. @@ -901,13 +854,6 @@ class Wav2Vec2BertForCTC(Wav2Vec2ConformerForCTC): ) -@add_start_docstrings( - """ - Wav2Vec2Bert Model with a sequence classification head on top (a linear layer over the pooled output) for - tasks like SUPERB Keyword Spotting. - """, - WAV2VEC2_BERT_START_DOCSTRING, -) class Wav2Vec2BertForSequenceClassification(Wav2Vec2ForSequenceClassification): def __init__(self, config): super().__init__(config) @@ -926,14 +872,6 @@ class Wav2Vec2BertForSequenceClassification(Wav2Vec2ForSequenceClassification): for param in self.wav2vec2_bert.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_BASE_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForSequenceClassification.forward with Wav2Vec2->Wav2Vec2Bert,wav2vec2->wav2vec2_bert,WAV_2_VEC_2->WAV2VEC2_BERT, input_values->input_features def forward( self, input_features: Optional[torch.Tensor], @@ -944,6 +882,11 @@ class Wav2Vec2BertForSequenceClassification(Wav2Vec2ForSequenceClassification): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2BertProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -997,12 +940,6 @@ class Wav2Vec2BertForSequenceClassification(Wav2Vec2ForSequenceClassification): ) -@add_start_docstrings( - """ - Wav2Vec2Bert Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAV2VEC2_BERT_START_DOCSTRING, -) class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2ConformerForAudioFrameClassification): def __init__(self, config): super().__init__(config) @@ -1010,14 +947,6 @@ class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2ConformerForAudioFrameClas def freeze_feature_encoder(self): raise AttributeError("Not needed for Wav2Vec2Bert") - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_BASE_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - # Copied from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer.Wav2Vec2ConformerForAudioFrameClassification.forward with wav2vec2_conformer->wav2vec2_bert, input_values->input_features def forward( self, input_features: Optional[torch.Tensor], @@ -1028,6 +957,11 @@ class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2ConformerForAudioFrameClas return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1072,12 +1006,6 @@ class Wav2Vec2BertForAudioFrameClassification(Wav2Vec2ConformerForAudioFrameClas ) -@add_start_docstrings( - """ - Wav2Vec2Bert Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAV2VEC2_BERT_START_DOCSTRING, -) class Wav2Vec2BertForXVector(Wav2Vec2ConformerForXVector): def __init__(self, config): super().__init__(config) @@ -1085,14 +1013,6 @@ class Wav2Vec2BertForXVector(Wav2Vec2ConformerForXVector): def freeze_feature_encoder(self): raise AttributeError("Not needed for Wav2Vec2Bert") - @add_start_docstrings_to_model_forward(WAV2VEC2_BERT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_BASE_CHECKPOINT_FOR_DOC, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - # Copied from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer.Wav2Vec2ConformerForXVector.forward with wav2vec2_conformer->wav2vec2_bert, input_values->input_features def forward( self, input_features: Optional[torch.Tensor], @@ -1103,6 +1023,11 @@ class Wav2Vec2BertForXVector(Wav2Vec2ConformerForXVector): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_features`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py b/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py index e5a0f2d87b..fc660d06ac 100644 --- a/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py +++ b/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py @@ -26,24 +26,10 @@ from ...modeling_outputs import ( XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_peft_available, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, is_peft_available from .configuration_wav2vec2_conformer import Wav2Vec2ConformerConfig -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/wav2vec2-conformer-rope-large-960h-ft" - -# General docstring -_CONFIG_FOR_DOC = "Wav2Vec2ConformerConfig" - - @dataclass class Wav2Vec2ConformerForPreTrainingOutput(ModelOutput): """ @@ -880,12 +866,8 @@ class Wav2Vec2ConformerAdapterLayer(nn.Module): return hidden_states +@auto_docstring class Wav2Vec2ConformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Wav2Vec2ConformerConfig base_model_prefix = "wav2vec2_conformer" main_input_name = "input_values" @@ -1098,72 +1080,10 @@ def _compute_mask_indices( return spec_aug_mask -_EXPECTED_OUTPUT_SHAPE = [1, 292, 1024] - - -WAV2VEC2_CONFORMER_START_DOCSTRING = r""" - Wav2Vec2Conformer was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech - Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael - Auli. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`Wav2Vec2ConformerConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -WAV2VEC2_CONFORMER_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, such as - [wav2vec2-conformer-rel-pos-large](https://huggingface.co/facebook/wav2vec2-conformer-rel-pos-large), - `attention_mask` should **not** be passed to avoid degraded performance when doing batched inference. For - such models `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware - that these models also yield slightly different results depending on whether `input_values` is padded or - not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - Wav2Vec2ConformerBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare Wav2Vec2Conformer Model transformer outputting raw hidden-states without any specific head on top.", - WAV2VEC2_CONFORMER_START_DOCSTRING, -) +@auto_docstring class Wav2Vec2ConformerModel(Wav2Vec2ConformerPreTrainedModel): def __init__(self, config: Wav2Vec2ConformerConfig): super().__init__(config) @@ -1235,14 +1155,7 @@ class Wav2Vec2ConformerModel(Wav2Vec2ConformerPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Wav2Vec2ConformerBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1252,6 +1165,11 @@ class Wav2Vec2ConformerModel(Wav2Vec2ConformerPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Wav2Vec2ConformerBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1296,8 +1214,10 @@ class Wav2Vec2ConformerModel(Wav2Vec2ConformerPreTrainedModel): ) -@add_start_docstrings( - """Wav2Vec2Conformer Model with a quantizer and `VQ` head on top.""", WAV2VEC2_CONFORMER_START_DOCSTRING +@auto_docstring( + custom_intro=""" + Wav2Vec2Conformer Model with a quantizer and `VQ` head on top. + """ ) class Wav2Vec2ConformerForPreTraining(Wav2Vec2ConformerPreTrainedModel): def __init__(self, config: Wav2Vec2ConformerConfig): @@ -1347,8 +1267,7 @@ class Wav2Vec2ConformerForPreTraining(Wav2Vec2ConformerPreTrainedModel): logits = logits / temperature return logits - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Wav2Vec2ConformerForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1367,8 +1286,6 @@ class Wav2Vec2ConformerForPreTraining(Wav2Vec2ConformerPreTrainedModel): Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss. Required input for pre-training. - Returns: - Example: ```python @@ -1512,17 +1429,20 @@ class Wav2Vec2ConformerForPreTraining(Wav2Vec2ConformerPreTrainedModel): _HIDDEN_STATES_START_POSITION = 2 -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 64.21 - -@add_start_docstrings( - """Wav2Vec2Conformer Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAV2VEC2_CONFORMER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + Wav2Vec2Conformer Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) class Wav2Vec2ConformerForCTC(Wav2Vec2ConformerPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.wav2vec2_conformer = Wav2Vec2ConformerModel(config) @@ -1552,14 +1472,7 @@ class Wav2Vec2ConformerForCTC(Wav2Vec2ConformerPreTrainedModel): """ self.wav2vec2_conformer.feature_extractor._freeze_parameters() - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1631,12 +1544,11 @@ class Wav2Vec2ConformerForCTC(Wav2Vec2ConformerPreTrainedModel): ) -@add_start_docstrings( +@auto_docstring( + custom_intro=""" + Wav2Vec2Conformer Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like + SUPERB Keyword Spotting. """ - Wav2Vec2Conformer Model with a sequence classification head on top (a linear layer over the pooled output) for - tasks like SUPERB Keyword Spotting. - """, - WAV2VEC2_CONFORMER_START_DOCSTRING, ) class Wav2Vec2ConformerForSequenceClassification(Wav2Vec2ConformerPreTrainedModel): def __init__(self, config): @@ -1671,13 +1583,7 @@ class Wav2Vec2ConformerForSequenceClassification(Wav2Vec2ConformerPreTrainedMode for param in self.wav2vec2_conformer.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1688,6 +1594,11 @@ class Wav2Vec2ConformerForSequenceClassification(Wav2Vec2ConformerPreTrainedMode labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1741,12 +1652,7 @@ class Wav2Vec2ConformerForSequenceClassification(Wav2Vec2ConformerPreTrainedMode ) -@add_start_docstrings( - """ - Wav2Vec2Conformer Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAV2VEC2_CONFORMER_START_DOCSTRING, -) +@auto_docstring class Wav2Vec2ConformerForAudioFrameClassification(Wav2Vec2ConformerPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1779,13 +1685,7 @@ class Wav2Vec2ConformerForAudioFrameClassification(Wav2Vec2ConformerPreTrainedMo for param in self.wav2vec2_conformer.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1796,6 +1696,11 @@ class Wav2Vec2ConformerForAudioFrameClassification(Wav2Vec2ConformerPreTrainedMo return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1895,11 +1800,10 @@ class TDNNLayer(nn.Module): return hidden_states -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Wav2Vec2Conformer Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAV2VEC2_CONFORMER_START_DOCSTRING, + """ ) class Wav2Vec2ConformerForXVector(Wav2Vec2ConformerPreTrainedModel): def __init__(self, config): @@ -1951,13 +1855,7 @@ class Wav2Vec2ConformerForXVector(Wav2Vec2ConformerPreTrainedModel): return input_lengths - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1968,6 +1866,11 @@ class Wav2Vec2ConformerForXVector(Wav2Vec2ConformerPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2ConformerProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/wav2vec2_conformer/modular_wav2vec2_conformer.py b/src/transformers/models/wav2vec2_conformer/modular_wav2vec2_conformer.py index c2d101385f..c95280ce68 100644 --- a/src/transformers/models/wav2vec2_conformer/modular_wav2vec2_conformer.py +++ b/src/transformers/models/wav2vec2_conformer/modular_wav2vec2_conformer.py @@ -8,23 +8,9 @@ from torch import nn from ...activations import ACT2FN from ...integrations.deepspeed import is_deepspeed_zero3_enabled from ...integrations.fsdp import is_fsdp_managed_module -from ...modeling_outputs import ( - BaseModelOutput, - CausalLMOutput, - SequenceClassifierOutput, - TokenClassifierOutput, - Wav2Vec2BaseModelOutput, - XVectorOutput, -) +from ...modeling_outputs import BaseModelOutput, Wav2Vec2BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from ..wav2vec2.modeling_wav2vec2 import ( Wav2Vec2Adapter, Wav2Vec2AdapterLayer, @@ -47,17 +33,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 2 -# General docstring -_CONFIG_FOR_DOC = "Wav2Vec2ConformerConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "facebook/wav2vec2-conformer-rope-large-960h-ft" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 1024] - -# CTC docstring -_CTC_EXPECTED_OUTPUT = "'MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'" -_CTC_EXPECTED_LOSS = 64.21 - @dataclass class Wav2Vec2ConformerForPreTrainingOutput(ModelOutput): @@ -584,12 +559,8 @@ class Wav2Vec2ConformerAdapterLayer(Wav2Vec2AdapterLayer): pass +@auto_docstring class Wav2Vec2ConformerPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = Wav2Vec2ConformerConfig base_model_prefix = "wav2vec2_conformer" main_input_name = "input_values" @@ -685,51 +656,10 @@ class Wav2Vec2ConformerPreTrainedModel(PreTrainedModel): WAV2VEC2_CONFORMER_START_DOCSTRING = None # will be automatically redefined -WAV2VEC2_CONFORMER_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, such as - [wav2vec2-conformer-rel-pos-large](https://huggingface.co/facebook/wav2vec2-conformer-rel-pos-large), - `attention_mask` should **not** be passed to avoid degraded performance when doing batched inference. For - such models `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware - that these models also yield slightly different results depending on whether `input_values` is padded or - not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" Wav2Vec2ConformerBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare Wav2Vec2Conformer Model transformer outputting raw hidden-states without any specific head on top.", - WAV2VEC2_CONFORMER_START_DOCSTRING, -) class Wav2Vec2ConformerModel(Wav2Vec2ConformerPreTrainedModel, Wav2Vec2Model): def __init__(self, config: Wav2Vec2ConformerConfig): Wav2Vec2ConformerPreTrainedModel.__init__(config) @@ -751,21 +681,7 @@ class Wav2Vec2ConformerModel(Wav2Vec2ConformerPreTrainedModel, Wav2Vec2Model): def freeze_feature_extractor(self): raise AttributeError("Not needed for Wav2Vec2Conformer") - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Wav2Vec2ConformerBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - -@add_start_docstrings( - """Wav2Vec2Conformer Model with a quantizer and `VQ` head on top.""", WAV2VEC2_CONFORMER_START_DOCSTRING -) class Wav2Vec2ConformerForPreTraining(Wav2Vec2ForPreTraining): def __init__(self, config: Wav2Vec2ConformerConfig): super().__init__(config) @@ -773,18 +689,15 @@ class Wav2Vec2ConformerForPreTraining(Wav2Vec2ForPreTraining): def freeze_feature_extractor(self): raise AttributeError("Not needed for Wav2Vec2Conformer") - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Wav2Vec2ConformerForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) - def forward(self, **super_kwargs) -> Union[Tuple, Wav2Vec2ConformerForPreTrainingOutput]: - return super().forward(**super_kwargs) - -@add_start_docstrings( - """Wav2Vec2Conformer Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAV2VEC2_CONFORMER_START_DOCSTRING, -) class Wav2Vec2ConformerForCTC(Wav2Vec2ForCTC): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`UniSpeechSatForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) def tie_weights(self): @@ -796,25 +709,7 @@ class Wav2Vec2ConformerForCTC(Wav2Vec2ForCTC): def freeze_base_model(self): raise AttributeError("Not needed for Wav2Vec2Conformer") - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - -@add_start_docstrings( - """ - Wav2Vec2Conformer Model with a sequence classification head on top (a linear layer over the pooled output) for - tasks like SUPERB Keyword Spotting. - """, - WAV2VEC2_CONFORMER_START_DOCSTRING, -) class Wav2Vec2ConformerForSequenceClassification(Wav2Vec2ForSequenceClassification): def __init__(self, config): super().__init__(config) @@ -822,23 +717,7 @@ class Wav2Vec2ConformerForSequenceClassification(Wav2Vec2ForSequenceClassificati def freeze_feature_extractor(self): raise AttributeError("Not needed for Wav2Vec2Conformer") - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - -@add_start_docstrings( - """ - Wav2Vec2Conformer Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAV2VEC2_CONFORMER_START_DOCSTRING, -) class Wav2Vec2ConformerForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification): def __init__(self, config): super().__init__(config) @@ -846,23 +725,7 @@ class Wav2Vec2ConformerForAudioFrameClassification(Wav2Vec2ForAudioFrameClassifi def freeze_feature_extractor(self): raise AttributeError("Not needed for Wav2Vec2Conformer") - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - -@add_start_docstrings( - """ - Wav2Vec2Conformer Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAV2VEC2_CONFORMER_START_DOCSTRING, -) class Wav2Vec2ConformerForXVector(Wav2Vec2ForXVector): def __init__(self, config): super().__init__(config) @@ -870,16 +733,6 @@ class Wav2Vec2ConformerForXVector(Wav2Vec2ForXVector): def freeze_feature_extractor(self): raise AttributeError("Not needed for Wav2Vec2Conformer") - @add_start_docstrings_to_model_forward(WAV2VEC2_CONFORMER_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - __all__ = [ "Wav2Vec2ConformerForAudioFrameClassification", diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py index 1c3c09d1a7..90330c2645 100755 --- a/src/transformers/models/wavlm/modeling_wavlm.py +++ b/src/transformers/models/wavlm/modeling_wavlm.py @@ -26,22 +26,12 @@ from ...modeling_outputs import ( XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_peft_available, - logging, -) +from ...utils import auto_docstring, is_peft_available, logging from .configuration_wavlm import WavLMConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "patrickvonplaten/wavlm-libri-clean-100h-base-plus" - -_CONFIG_FOR_DOC = "WavLMConfig" - class WavLMSamePadLayer(nn.Module): def __init__(self, num_conv_pos_embeddings): @@ -619,12 +609,8 @@ class WavLMGumbelVectorQuantizer(nn.Module): return codevectors, perplexity +@auto_docstring class WavLMPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = WavLMConfig base_model_prefix = "wavlm" main_input_name = "input_values" @@ -993,71 +979,10 @@ def _compute_mask_indices( return spec_aug_mask -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - - -WAVLM_START_DOCSTRING = r""" - WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled - Data](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo - Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, - Jian Wu, Michael Zeng, Xiangzhan Yu, Furu Wei. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`WavLMConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -WAVLM_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - WavLMBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare WavLM Model transformer outputting raw hidden-states without any specific head on top.", - WAVLM_START_DOCSTRING, -) +@auto_docstring class WavLMModel(WavLMPreTrainedModel): def __init__(self, config: WavLMConfig): super().__init__(config) @@ -1144,14 +1069,7 @@ class WavLMModel(WavLMPreTrainedModel): return hidden_states - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=WavLMBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1161,6 +1079,11 @@ class WavLMModel(WavLMPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, WavLMBaseModelOutput]: + r""" + mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict + masked extracted features in *config.proj_codevector_dim* space. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1207,16 +1130,20 @@ class WavLMModel(WavLMPreTrainedModel): _HIDDEN_STATES_START_POSITION = 2 -_CTC_EXPECTED_OUTPUT = "'mister quilter is the aposle of the middle classes and we are glad to welcome his gospel'" -_CTC_EXPECTED_LOSS = 12.51 - -@add_start_docstrings( - """WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAVLM_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + """ ) class WavLMForCTC(WavLMPreTrainedModel): def __init__(self, config, target_lang: Optional[str] = None): + r""" + target_lang (`str`, *optional*): + Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or + adapter..bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by + default. + """ super().__init__(config) self.wavlm = WavLMModel(config) @@ -1287,14 +1214,7 @@ class WavLMForCTC(WavLMPreTrainedModel): for param in self.wavlm.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1366,12 +1286,11 @@ class WavLMForCTC(WavLMPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - WAVLM_START_DOCSTRING, + """ ) class WavLMForSequenceClassification(WavLMPreTrainedModel): def __init__(self, config): @@ -1418,13 +1337,7 @@ class WavLMForSequenceClassification(WavLMPreTrainedModel): for param in self.wavlm.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1435,6 +1348,11 @@ class WavLMForSequenceClassification(WavLMPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1488,16 +1406,7 @@ class WavLMForSequenceClassification(WavLMPreTrainedModel): ) -_FRAME_CLASS_CHECKPOINT = "microsoft/wavlm-base-plus-sd" -_FRAME_EXPECTED_OUTPUT = [0, 0] - - -@add_start_docstrings( - """ - WavLM Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAVLM_START_DOCSTRING, -) +@auto_docstring class WavLMForAudioFrameClassification(WavLMPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1542,14 +1451,7 @@ class WavLMForAudioFrameClassification(WavLMPreTrainedModel): for param in self.wavlm.parameters(): param.requires_grad = False - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_FRAME_CLASS_CHECKPOINT, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_FRAME_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1560,6 +1462,11 @@ class WavLMForAudioFrameClassification(WavLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1659,15 +1566,10 @@ class TDNNLayer(nn.Module): return hidden_states -_XVECTOR_CHECKPOINT = "microsoft/wavlm-base-plus-sv" -_XVECTOR_EXPECTED_OUTPUT = 0.97 - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAVLM_START_DOCSTRING, + """ ) class WavLMForXVector(WavLMPreTrainedModel): def __init__(self, config): @@ -1731,14 +1633,7 @@ class WavLMForXVector(WavLMPreTrainedModel): return input_lengths - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_XVECTOR_CHECKPOINT, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_XVECTOR_EXPECTED_OUTPUT, - ) + @auto_docstring def forward( self, input_values: Optional[torch.Tensor], @@ -1749,6 +1644,11 @@ class WavLMForXVector(WavLMPreTrainedModel): labels: Optional[torch.Tensor] = None, ) -> Union[Tuple, XVectorOutput]: r""" + input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If diff --git a/src/transformers/models/wavlm/modular_wavlm.py b/src/transformers/models/wavlm/modular_wavlm.py index 9ae9170fec..b74d9fc703 100644 --- a/src/transformers/models/wavlm/modular_wavlm.py +++ b/src/transformers/models/wavlm/modular_wavlm.py @@ -7,16 +7,9 @@ import torch.nn.functional as F from ...integrations.deepspeed import is_deepspeed_zero3_enabled from ...integrations.fsdp import is_fsdp_managed_module -from ...modeling_outputs import ( - BaseModelOutput, - CausalLMOutput, - SequenceClassifierOutput, - TokenClassifierOutput, - Wav2Vec2BaseModelOutput, - XVectorOutput, -) +from ...modeling_outputs import BaseModelOutput, Wav2Vec2BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import logging from ..wav2vec2.modeling_wav2vec2 import ( Wav2Vec2FeatureProjection, Wav2Vec2FeedForward, @@ -33,20 +26,6 @@ from .configuration_wavlm import WavLMConfig logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "WavLMConfig" - -_CHECKPOINT_FOR_DOC = "patrickvonplaten/wavlm-libri-clean-100h-base-plus" -_EXPECTED_OUTPUT_SHAPE = [1, 292, 768] - -_CTC_EXPECTED_OUTPUT = "'mister quilter is the aposle of the middle classes and we are glad to welcome his gospel'" -_CTC_EXPECTED_LOSS = 12.51 - -_FRAME_CLASS_CHECKPOINT = "microsoft/wavlm-base-plus-sd" -_FRAME_EXPECTED_OUTPUT = [0, 0] - -_XVECTOR_CHECKPOINT = "microsoft/wavlm-base-plus-sv" -_XVECTOR_EXPECTED_OUTPUT = 0.97 - class WavLMPositionalConvEmbedding(Wav2Vec2PositionalConvEmbedding): pass @@ -542,11 +521,6 @@ class WavLMGumbelVectorQuantizer(nn.Module): class WavLMPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = WavLMConfig base_model_prefix = "wavlm" main_input_name = "input_values" @@ -597,155 +571,27 @@ class WavLMPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel): raise AttributeError("Not needed for WavLM") -WAVLM_START_DOCSTRING = r""" - WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled - Data](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo - Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, - Jian Wu, Michael Zeng, Xiangzhan Yu, Furu Wei. - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving etc.). - - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`WavLMConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -WAVLM_INPUTS_DOCSTRING = r""" - Args: - input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): - Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file - into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install - soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and - conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, - 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - - - `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == - True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should - **not** be passed to avoid degraded performance when doing batched inference. For such models - `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these - models also yield slightly different results depending on whether `input_values` is padded or not. - - - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - WavLMBaseModelOutput = Wav2Vec2BaseModelOutput -@add_start_docstrings( - "The bare WavLM Model transformer outputting raw hidden-states without any specific head on top.", - WAVLM_START_DOCSTRING, -) class WavLMModel(Wav2Vec2Model): - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=WavLMBaseModelOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) - def forward(self, **super_kwargs): - return super().forward(**super_kwargs) - - -@add_start_docstrings( - """WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""", - WAVLM_START_DOCSTRING, -) -class WavLMForCTC(Wav2Vec2ForCTC): - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_CTC_EXPECTED_OUTPUT, - expected_loss=_CTC_EXPECTED_LOSS, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - - -@add_start_docstrings( - """ - WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like - SUPERB Keyword Spotting. - """, - WAVLM_START_DOCSTRING, -) -class WavLMForSequenceClassification(Wav2Vec2ForSequenceClassification): - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - - -@add_start_docstrings( - """ - WavLM Model with a frame classification head on top for tasks like Speaker Diarization. - """, - WAVLM_START_DOCSTRING, -) -class WavLMForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification): - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_FRAME_CLASS_CHECKPOINT, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_FRAME_EXPECTED_OUTPUT, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) - - -@add_start_docstrings( - """ - WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification. - """, - WAVLM_START_DOCSTRING, -) -class WavLMForXVector(Wav2Vec2ForXVector): pass - @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_XVECTOR_CHECKPOINT, - output_type=XVectorOutput, - config_class=_CONFIG_FOR_DOC, - modality="audio", - expected_output=_XVECTOR_EXPECTED_OUTPUT, - ) - def forward(self, **super_kwargs): - super().forward(**super_kwargs) + +class WavLMForCTC(Wav2Vec2ForCTC): + pass + + +class WavLMForSequenceClassification(Wav2Vec2ForSequenceClassification): + pass + + +class WavLMForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification): + pass + + +class WavLMForXVector(Wav2Vec2ForXVector): + pass __all__ = [ diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index 5ef38be037..59a999c2ab 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -37,13 +37,7 @@ from ...modeling_outputs import ( SequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_flex_attn_available, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, is_torch_flex_attn_available, logging from .configuration_whisper import WhisperConfig from .generation_whisper import WhisperGenerationMixin @@ -61,9 +55,6 @@ logger = logging.get_logger(__name__) _HIDDEN_STATES_START_POSITION = 1 -_CONFIG_FOR_DOC = "WhisperConfig" -_CHECKPOINT_FOR_DOC = "openai/whisper-tiny" - def sinusoids(length: int, channels: int, max_timescale: float = 10000) -> torch.Tensor: """Returns sinusoids for positional embedding""" @@ -765,6 +756,7 @@ class WhisperDecoderLayer(nn.Module): return outputs +@auto_docstring class WhisperPreTrainedModel(PreTrainedModel): config_class = WhisperConfig base_model_prefix = "model" @@ -804,142 +796,6 @@ class WhisperPreTrainedModel(PreTrainedModel): return input_lengths -WHISPER_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`WhisperConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -WHISPER_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): - Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by - loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via - the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing *SpecAugment* data augmentation on padding token indices. Mask values selected in - `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - Whisper uses the `decoder_start_token_id` as the starting token for `decoder_input_ids` generation. If - `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see - `past_key_values`). - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should read - [`modeling_whisper._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the BART - paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - past_key_values (`EncoderDecoderCache` or `tuple(tuple(torch.FloatTensor))`, *optional*): - Pre-computed hidden-states that can be used to speed up auto-regressive (sequential) decoding. There are - four sets of pre-computed hidden-states: key and values states in the self-attention blocks (2) and - in the cross-attention blocks (2). The `past_key_values` are returned when `use_cache=True` is passed or - when `config.use_cache=True` - - Two formats are allowed: - - An [`~cache_utils.EncoderDecoderCache`] instance; - - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the cache - in the correct position and to infer the complete sequence length. -""" - -WHISPER_ENCODER_INPUTS_DOCSTRING = r""" - Args: - input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): - Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by - loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via - the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the - [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a - tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] - head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - class WhisperEncoder(WhisperPreTrainedModel): """ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a @@ -1498,10 +1354,7 @@ class WhisperDecoder(WhisperPreTrainedModel): return causal_mask -@add_start_docstrings( - "The bare Whisper Model outputting raw hidden-states without any specific head on top.", - WHISPER_START_DOCSTRING, -) +@auto_docstring class WhisperModel(WhisperPreTrainedModel): def __init__(self, config: WhisperConfig): super().__init__(config) @@ -1573,8 +1426,7 @@ class WhisperModel(WhisperPreTrainedModel): return input_features - @add_start_docstrings_to_model_forward(WHISPER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_features: Optional[torch.FloatTensor] = None, @@ -1595,7 +1447,40 @@ class WhisperModel(WhisperPreTrainedModel): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqModelOutput]: r""" - Returns: + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Whisper uses the `decoder_start_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_whisper._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the BART + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) Example: ```python @@ -1670,9 +1555,10 @@ class WhisperModel(WhisperPreTrainedModel): ) -@add_start_docstrings( - "The Whisper Model with a language modeling head. Can be used for automatic speech recognition.", - WHISPER_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + The Whisper Model with a language modeling head. Can be used for automatic speech recognition. + """ ) class WhisperForConditionalGeneration(WhisperGenerationMixin, WhisperPreTrainedModel): base_model_prefix = "model" @@ -1709,8 +1595,7 @@ class WhisperForConditionalGeneration(WhisperGenerationMixin, WhisperPreTrainedM """ self.model.encoder._freeze_parameters() - @add_start_docstrings_to_model_forward(WHISPER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_features: Optional[torch.FloatTensor] = None, @@ -1732,12 +1617,44 @@ class WhisperForConditionalGeneration(WhisperGenerationMixin, WhisperPreTrainedM cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Whisper uses the `decoder_start_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read + [`modeling_whisper._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the BART + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. `sequence_length` should be smaller than or equal to `config.max_target_positions`. + decoder_position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. - Returns: + [What are position IDs?](../glossary#position-ids) Example: @@ -1840,11 +1757,10 @@ class WhisperDecoderWrapper(WhisperPreTrainedModel): return self.decoder(*args, **kwargs) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Whisper decoder with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - WHISPER_START_DOCSTRING, + """ ) class WhisperForCausalLM(WhisperPreTrainedModel, GenerationMixin): _tied_weights_keys = ["proj_out.weight"] @@ -1878,7 +1794,7 @@ class WhisperForCausalLM(WhisperPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1896,63 +1812,17 @@ class WhisperForCausalLM(WhisperPreTrainedModel, GenerationMixin): cache_position: Optional[torch.LongTensor] = None, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - encoder_outputs (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains - pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If - `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. It is used to update the cache - in the correct position and to infer the complete sequence length. - - Returns: + encoder_outputs (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + if the model is configured as a decoder. + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -2036,12 +1906,11 @@ class WhisperForCausalLM(WhisperPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" Whisper Encoder Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. - """, - WHISPER_ENCODER_INPUTS_DOCSTRING, + """ ) class WhisperForAudioClassification(WhisperPreTrainedModel): def __init__(self, config): @@ -2070,8 +1939,7 @@ class WhisperForAudioClassification(WhisperPreTrainedModel): def set_input_embeddings(self, value: nn.Module): self.encoder.set_input_embeddings(value) - @add_start_docstrings_to_model_forward(WHISPER_ENCODER_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_features: Optional[torch.LongTensor] = None, @@ -2083,13 +1951,17 @@ class WhisperForAudioClassification(WhisperPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): + Float values mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - Returns: - Example: ```python diff --git a/src/transformers/models/x_clip/modeling_x_clip.py b/src/transformers/models/x_clip/modeling_x_clip.py index a6a3655961..5f07ab0cc5 100644 --- a/src/transformers/models/x_clip/modeling_x_clip.py +++ b/src/transformers/models/x_clip/modeling_x_clip.py @@ -28,10 +28,8 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...utils import ( ModelOutput, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, logging, - replace_return_docstrings, torch_int, ) from .configuration_x_clip import XCLIPConfig, XCLIPTextConfig, XCLIPVisionConfig @@ -39,8 +37,6 @@ from .configuration_x_clip import XCLIPConfig, XCLIPTextConfig, XCLIPVisionConfi logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "microsoft/xclip-base-patch32" - # contrastive loss function, adapted from # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/CLIP.html @@ -508,12 +504,8 @@ class XCLIPVisionEncoderLayer(nn.Module): return outputs +@auto_docstring class XCLIPPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = XCLIPConfig base_model_prefix = "x_clip" supports_gradient_checkpointing = True @@ -566,106 +558,6 @@ class XCLIPPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -X_CLIP_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`XCLIPConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -X_CLIP_TEXT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -X_CLIP_VISION_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -X_CLIP_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. - return_loss (`bool`, *optional*): - Whether or not to return the contrastive loss. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - interpolate_pos_encoding (`bool`, *optional*, defaults `False`): - Whether to interpolate the pre-trained position encodings. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - # Copied from transformers.models.altclip.modeling_altclip.AltCLIPEncoder with AltCLIP->XCLIP class XCLIPEncoder(nn.Module): """ @@ -773,8 +665,7 @@ class XCLIPTextTransformer(nn.Module): self.encoder = XCLIPEncoder(config) self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(X_CLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=XCLIPTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -784,10 +675,6 @@ class XCLIPTextTransformer(nn.Module): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -854,8 +741,7 @@ class XCLIPTextModel(XCLIPPreTrainedModel): def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value - @add_start_docstrings_to_model_forward(X_CLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=XCLIPTextConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -866,8 +752,6 @@ class XCLIPTextModel(XCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -1004,8 +888,7 @@ class XCLIPVisionTransformer(nn.Module): self.encoder = XCLIPVisionEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) - @add_start_docstrings_to_model_forward(X_CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=XCLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1014,10 +897,6 @@ class XCLIPVisionTransformer(nn.Module): interpolate_pos_encoding: bool = False, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1062,8 +941,7 @@ class XCLIPVisionModel(XCLIPPreTrainedModel): def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(X_CLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=XCLIPVisionConfig) + @auto_docstring def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1072,8 +950,6 @@ class XCLIPVisionModel(XCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: r""" - Returns: - Examples: ```python @@ -1292,7 +1168,7 @@ class XCLIPPromptGenerator(nn.Module): return self.alpha * text -@add_start_docstrings(X_CLIP_START_DOCSTRING) +@auto_docstring class XCLIPModel(XCLIPPreTrainedModel): config_class = XCLIPConfig @@ -1339,7 +1215,7 @@ class XCLIPModel(XCLIPPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(X_CLIP_TEXT_INPUTS_DOCSTRING) + @auto_docstring def get_text_features( self, input_ids: Optional[torch.Tensor] = None, @@ -1386,7 +1262,7 @@ class XCLIPModel(XCLIPPreTrainedModel): return text_embeds - @add_start_docstrings_to_model_forward(X_CLIP_VISION_INPUTS_DOCSTRING) + @auto_docstring def get_video_features( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -1501,8 +1377,7 @@ class XCLIPModel(XCLIPPreTrainedModel): return video_embeds - @add_start_docstrings_to_model_forward(X_CLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=XCLIPOutput, config_class=XCLIPConfig) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1516,7 +1391,8 @@ class XCLIPModel(XCLIPPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, XCLIPOutput]: r""" - Returns: + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Examples: diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index feb91b46f3..1b9c14ddca 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -26,102 +26,12 @@ from ...generation import GenerationMixin from ...modeling_attn_mask_utils import _prepare_4d_attention_mask, _prepare_4d_causal_attention_mask from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_xglm import XGLMConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/xglm-564M" -_CONFIG_FOR_DOC = "XGLMConfig" - - -XGLM_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`XGLMConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -XGLM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of - the decoder. - encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape - `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape - `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention - blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - # Copied from transformers.models.bart.modeling_bart.BartScaledWordEmbedding with Bart->XGLM class XGLMScaledWordEmbedding(nn.Embedding): @@ -463,6 +373,7 @@ class XGLMDecoderLayer(nn.Module): return outputs +@auto_docstring class XGLMPreTrainedModel(PreTrainedModel): config_class = XGLMConfig base_model_prefix = "model" @@ -481,20 +392,13 @@ class XGLMPreTrainedModel(PreTrainedModel): module.weight.data[module.padding_idx].zero_() -@add_start_docstrings( - "The bare XGLM Model transformer outputting raw hidden-states without any specific head on top.", - XGLM_START_DOCSTRING, -) +@auto_docstring class XGLMModel(XGLMPreTrainedModel): - """ - Transformer decoder consisting of *config.num_layers* layers. Each layer is a [`XGLMDecoderLayer`] - - Args: - config: XGLMConfig - embed_tokens (nn.Embedding): output embedding - """ - def __init__(self, config: XGLMConfig, embed_tokens: Optional[nn.Embedding] = None): + r""" + embed_tokens (`nn.Embedding`, *optional*): + output embeddings + """ super().__init__(config) self.dropout = config.dropout self.layerdrop = config.layerdrop @@ -527,12 +431,7 @@ class XGLMModel(XGLMPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -549,6 +448,24 @@ class XGLMModel(XGLMPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of + the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -690,12 +607,11 @@ class XGLMModel(XGLMPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The XGLM Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - XGLM_START_DOCSTRING, + """ ) class XGLMForCausalLM(XGLMPreTrainedModel, GenerationMixin): base_model_prefix = "model" @@ -721,12 +637,7 @@ class XGLMForCausalLM(XGLMPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - @add_start_docstrings_to_model_forward(XGLM_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -746,6 +657,22 @@ class XGLMForCausalLM(XGLMPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of + the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored diff --git a/src/transformers/models/xlm/modeling_xlm.py b/src/transformers/models/xlm/modeling_xlm.py index 16f1d4ec3f..df017e27b1 100755 --- a/src/transformers/models/xlm/modeling_xlm.py +++ b/src/transformers/models/xlm/modeling_xlm.py @@ -38,22 +38,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_xlm import XLMConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "FacebookAI/xlm-mlm-en-2048" -_CONFIG_FOR_DOC = "XLMConfig" - def create_sinusoidal_embeddings(n_pos, dim, out): position_enc = np.array([[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]) @@ -310,7 +300,7 @@ class XLMSQuADHead(nn.Module): self.end_logits = XLMPoolerEndLogits(config) self.answer_class = XLMPoolerAnswerClass(config) - @replace_return_docstrings(output_type=XLMSquadHeadOutput, config_class=XLMConfig) + @auto_docstring def forward( self, hidden_states: torch.FloatTensor, @@ -321,25 +311,20 @@ class XLMSQuADHead(nn.Module): p_mask: Optional[torch.FloatTensor] = None, return_dict: bool = False, ) -> Union[XLMSquadHeadOutput, Tuple[torch.FloatTensor]]: - """ - Args: - hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): - Final hidden states of the model on the sequence tokens. - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Positions of the first token for the labeled span. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Positions of the last token for the labeled span. - cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Position of the CLS token for each sentence in the batch. If `None`, takes the last token. - is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Whether the question has a possible answer in the paragraph or not. - p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): - Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token - should be masked. - return_dict (`bool`, *optional*, defaults to `False`): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - - Returns: + r""" + hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`): + Final hidden states of the model on the sequence tokens. + start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Positions of the first token for the labeled span. + end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Positions of the last token for the labeled span. + cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Position of the CLS token for each sentence in the batch. If `None`, takes the last token. + is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Whether the question has a possible answer in the paragraph or not. + p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*): + Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token + should be masked. """ start_logits = self.start_logits(hidden_states, p_mask=p_mask) @@ -624,12 +609,8 @@ class TransformerFFN(nn.Module): return x +@auto_docstring class XLMPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = XLMConfig load_tf_weights = None base_model_prefix = "transformer" @@ -711,95 +692,7 @@ class XLMForQuestionAnsweringOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor, ...]] = None -XLM_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`XLMConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -XLM_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - langs (`torch.LongTensor` of shape `({0})`, *optional*): - A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are - languages ids which can be obtained from the language names by using two conversion mappings provided in - the configuration of the model (only provided for multilingual models). More precisely, the *language name - to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the - *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). - - See usage examples detailed in the [multilingual documentation](../multilingual). - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Length of each sentence that can be used to avoid performing attention on padding token indices. You can - also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in - `[0, ..., input_ids.size(-1)]`. - cache (`Dict[str, torch.FloatTensor]`, *optional*): - Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the - attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential - decoding. - - The dictionary object will be modified in-place during the forward pass to add newly computed - hidden-states. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare XLM Model transformer outputting raw hidden-states without any specific head on top.", - XLM_START_DOCSTRING, -) +@auto_docstring class XLMModel(XLMPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -885,12 +778,7 @@ class XLMModel(XLMPreTrainedModel): for layer, heads in heads_to_prune.items(): self.attentions[layer].prune_heads(heads) - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -907,6 +795,27 @@ class XLMModel(XLMPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, # Dummy kwargs for now ) -> Union[Tuple, BaseModelOutput]: + r""" + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1071,12 +980,11 @@ class XLMPredLayer(nn.Module): return outputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The XLM Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - XLM_START_DOCSTRING, + """ ) class XLMWithLMHeadModel(XLMPreTrainedModel, GenerationMixin): _tied_weights_keys = ["pred_layer.proj.weight"] @@ -1110,13 +1018,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel, GenerationMixin): langs = None return {"input_ids": input_ids, "langs": langs} - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1135,6 +1037,25 @@ class XLMWithLMHeadModel(XLMPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, MaskedLMOutput]: r""" + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` @@ -1172,12 +1093,11 @@ class XLMWithLMHeadModel(XLMPreTrainedModel, GenerationMixin): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLM Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - XLM_START_DOCSTRING, + """ ) class XLMForSequenceClassification(XLMPreTrainedModel): def __init__(self, config): @@ -1191,12 +1111,7 @@ class XLMForSequenceClassification(XLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1214,6 +1129,25 @@ class XLMForSequenceClassification(XLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, SequenceClassifierOutput]: r""" + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1274,12 +1208,11 @@ class XLMForSequenceClassification(XLMPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XLM_START_DOCSTRING, + """ ) class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): def __init__(self, config): @@ -1291,12 +1224,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1315,14 +1243,25 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1378,13 +1317,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): ) -@add_start_docstrings( - """ - XLM Model with a beam-search span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XLM_START_DOCSTRING, -) +@auto_docstring class XLMForQuestionAnswering(XLMPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1395,8 +1328,7 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=XLMForQuestionAnsweringOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1418,14 +1350,25 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, XLMForQuestionAnsweringOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels whether a question has an answer or no answer (SQuAD 2.0) cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): @@ -1435,8 +1378,6 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be masked. 0.0 mean token is not masked. - Returns: - Example: ```python @@ -1499,13 +1440,7 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): ) -@add_start_docstrings( - """ - XLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - XLM_START_DOCSTRING, -) +@auto_docstring class XLMForTokenClassification(XLMPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1518,12 +1453,7 @@ class XLMForTokenClassification(XLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1541,6 +1471,25 @@ class XLMForTokenClassification(XLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, TokenClassifierOutput]: r""" + langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1583,13 +1532,7 @@ class XLMForTokenClassification(XLMPreTrainedModel): ) -@add_start_docstrings( - """ - XLM Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - XLM_START_DOCSTRING, -) +@auto_docstring class XLMForMultipleChoice(XLMPreTrainedModel): def __init__(self, config, *inputs, **kwargs): super().__init__(config, *inputs, **kwargs) @@ -1601,12 +1544,7 @@ class XLMForMultipleChoice(XLMPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1624,6 +1562,49 @@ class XLMForMultipleChoice(XLMPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + langs (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are + languages ids which can be obtained from the language names by using two conversion mappings provided in + the configuration of the model (only provided for multilingual models). More precisely, the *language name + to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the + *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). + + See usage examples detailed in the [multilingual documentation](../multilingual). + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Length of each sentence that can be used to avoid performing attention on padding token indices. You can + also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in + `[0, ..., input_ids.size(-1)]`. + cache (`Dict[str, torch.FloatTensor]`, *optional*): + Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the + attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential + decoding. + + The dictionary object will be modified in-place during the forward pass to add newly computed + hidden-states. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See diff --git a/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py b/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py index dcc9bf2344..6a702985db 100644 --- a/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py +++ b/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py @@ -26,10 +26,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN, gelu from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - _prepare_4d_attention_mask_for_sdpa, - _prepare_4d_causal_attention_mask_for_sdpa, -) +from ...modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -42,22 +39,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - get_torch_version, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, get_torch_version, logging from .configuration_xlm_roberta import XLMRobertaConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "FacebookAI/xlm-roberta-base" -_CONFIG_FOR_DOC = "XLMRobertaConfig" - # Copied from transformers.models.roberta.modeling_roberta.RobertaEmbeddings with Roberta->XLMRoberta class XLMRobertaEmbeddings(nn.Module): @@ -687,13 +674,9 @@ class XLMRobertaPooler(nn.Module): return pooled_output +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaPreTrainedModel with Roberta->XLMRoberta class XLMRobertaPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = XLMRobertaConfig base_model_prefix = "roberta" supports_gradient_checkpointing = True @@ -720,93 +703,16 @@ class XLMRobertaPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -XLM_ROBERTA_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`XLMRobertaConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -XLM_ROBERTA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare XLM-RoBERTa Model transformer outputting raw hidden-states without any specific head on top.", - XLM_ROBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaModel with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaModel(XLMRobertaPreTrainedModel): - """ - - The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of - cross-attention is added between the self-attention layers, following the architecture described in [Attention is - all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, - Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. - - To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set - to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and - `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. - """ - _no_split_modules = ["XLMRobertaEmbeddings", "XLMRobertaLayer"] def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -835,12 +741,7 @@ class XLMRobertaModel(XLMRobertaPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -857,26 +758,6 @@ class XLMRobertaModel(XLMRobertaPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, target_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1004,9 +885,10 @@ class XLMRobertaModel(XLMRobertaPreTrainedModel): ) -@add_start_docstrings( - "XLM-RoBERTa Model with a `language modeling` head on top for CLM fine-tuning.", - XLM_ROBERTA_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + XLM-RoBERTa Model with a `language modeling` head on top for CLM fine-tuning. + """ ) # Copied from transformers.models.roberta.modeling_roberta.RobertaForCausalLM with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaForCausalLM(XLMRobertaPreTrainedModel, GenerationMixin): @@ -1030,8 +912,7 @@ class XLMRobertaForCausalLM(XLMRobertaPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1051,31 +932,19 @@ class XLMRobertaForCausalLM(XLMRobertaPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1149,10 +1018,7 @@ class XLMRobertaForCausalLM(XLMRobertaPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """XLM-RoBERTa Model with a `language modeling` head on top.""", - XLM_ROBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForMaskedLM with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaForMaskedLM(XLMRobertaPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1178,15 +1044,7 @@ class XLMRobertaForMaskedLM(XLMRobertaPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - expected_output="' Paris'", - expected_loss=0.1, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1203,12 +1061,19 @@ class XLMRobertaForMaskedLM(XLMRobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1279,12 +1144,11 @@ class XLMRobertaLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLM-RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - XLM_ROBERTA_START_DOCSTRING, + """ ) # Copied from transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaForSequenceClassification(XLMRobertaPreTrainedModel): @@ -1299,14 +1163,7 @@ class XLMRobertaForSequenceClassification(XLMRobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="cardiffnlp/twitter-roberta-base-emotion", - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="'optimism'", - expected_loss=0.08, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1321,6 +1178,15 @@ class XLMRobertaForSequenceClassification(XLMRobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1379,13 +1245,7 @@ class XLMRobertaForSequenceClassification(XLMRobertaPreTrainedModel): ) -@add_start_docstrings( - """ - XLM-RoBERTa Model with a multiple choice classification head on top (a linear layer on top of the pooled output and - a softmax) e.g. for RocStories/SWAG tasks. - """, - XLM_ROBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForMultipleChoice with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaForMultipleChoice(XLMRobertaPreTrainedModel): def __init__(self, config): @@ -1398,14 +1258,7 @@ class XLMRobertaForMultipleChoice(XLMRobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward( - XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1420,10 +1273,35 @@ class XLMRobertaForMultipleChoice(XLMRobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1474,13 +1352,7 @@ class XLMRobertaForMultipleChoice(XLMRobertaPreTrainedModel): ) -@add_start_docstrings( - """ - XLM-RoBERTa Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. - for Named-Entity-Recognition (NER) tasks. - """, - XLM_ROBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForTokenClassification with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaForTokenClassification(XLMRobertaPreTrainedModel): def __init__(self, config): @@ -1497,14 +1369,7 @@ class XLMRobertaForTokenClassification(XLMRobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="Jean-Baptiste/roberta-large-ner-english", - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="['O', 'ORG', 'ORG', 'O', 'O', 'O', 'O', 'O', 'LOC', 'O', 'LOC', 'LOC']", - expected_loss=0.01, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1519,6 +1384,15 @@ class XLMRobertaForTokenClassification(XLMRobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1583,13 +1457,7 @@ class XLMRobertaClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - XLM-RoBERTa Model with a span classification head on top for extractive question-answering tasks like SQuAD (a - linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XLM_ROBERTA_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.roberta.modeling_roberta.RobertaForQuestionAnswering with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA class XLMRobertaForQuestionAnswering(XLMRobertaPreTrainedModel): def __init__(self, config): @@ -1602,14 +1470,7 @@ class XLMRobertaForQuestionAnswering(XLMRobertaPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLM_ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint="deepset/roberta-base-squad2", - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_output="' puppet'", - expected_loss=0.86, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1625,14 +1486,15 @@ class XLMRobertaForQuestionAnswering(XLMRobertaPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value + >= 2. All the value in this tensor should be always < type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py index ad43c7903f..6040e40021 100644 --- a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py +++ b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py @@ -25,10 +25,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN, gelu from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - _prepare_4d_attention_mask_for_sdpa, - _prepare_4d_causal_attention_mask_for_sdpa, -) +from ...modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -41,22 +38,12 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - get_torch_version, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, get_torch_version, logging from .configuration_xlm_roberta_xl import XLMRobertaXLConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "facebook/xlm-roberta-xl" -_CONFIG_FOR_DOC = "XLMRobertaXLConfig" - class XLMRobertaXLEmbeddings(nn.Module): """ @@ -681,12 +668,8 @@ class XLMRobertaXLPooler(nn.Module): return pooled_output +@auto_docstring class XLMRobertaXLPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = XLMRobertaXLConfig base_model_prefix = "roberta" _no_split_modules = ["XLMRobertaXLEmbeddings", "XLMRobertaXLLayer"] @@ -712,82 +695,16 @@ class XLMRobertaXLPreTrainedModel(PreTrainedModel): module.bias.data.zero_() -XLM_ROBERTA_XL_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) - subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to - general usage and behavior. - - Parameters: - config ([`XLMRobertaXLConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -XLM_ROBERTA_XL_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See - [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input - IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare XLM-RoBERTa-XL Model transformer outputting raw hidden-states without any specific head on top.", - XLM_ROBERTA_XL_START_DOCSTRING, -) +@auto_docstring # Copied from transformers.models.bert.modeling_bert.BertModel with Bert->XLMRobertaXL, BERT->XLM_ROBERTA_XL class XLMRobertaXLModel(XLMRobertaXLPreTrainedModel): - """ - - The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of - cross-attention is added between the self-attention layers, following the architecture described in [Attention is - all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, - Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. - - To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set - to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and - `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. - """ - _no_split_modules = ["XLMRobertaXLEmbeddings", "XLMRobertaXLLayer"] def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -816,12 +733,7 @@ class XLMRobertaXLModel(XLMRobertaXLPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPoolingAndCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -838,26 +750,6 @@ class XLMRobertaXLModel(XLMRobertaXLPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: - r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, target_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -985,9 +877,10 @@ class XLMRobertaXLModel(XLMRobertaXLPreTrainedModel): ) -@add_start_docstrings( - """XLM-RoBERTa-XL Model with a `language modeling` head on top for CLM fine-tuning.""", - XLM_ROBERTA_XL_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + XLM-RoBERTa-XL Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class XLMRobertaXLForCausalLM(XLMRobertaXLPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1010,8 +903,7 @@ class XLMRobertaXLForCausalLM(XLMRobertaXLPreTrainedModel, GenerationMixin): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1031,29 +923,10 @@ class XLMRobertaXLForCausalLM(XLMRobertaXLPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: Example: @@ -1161,9 +1034,7 @@ class XLMRobertaXLForCausalLM(XLMRobertaXLPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """XLM-RoBERTa-XL Model with a `language modeling` head on top.""", XLM_ROBERTA_XL_START_DOCSTRING -) +@auto_docstring class XLMRobertaXLForMaskedLM(XLMRobertaXLPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1188,13 +1059,7 @@ class XLMRobertaXLForMaskedLM(XLMRobertaXLPreTrainedModel): self.lm_head.decoder = new_embeddings self.lm_head.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - mask="", - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1215,8 +1080,6 @@ class XLMRobertaXLForMaskedLM(XLMRobertaXLPreTrainedModel): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1284,12 +1147,11 @@ class XLMRobertaXLLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLM-RoBERTa-XL Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - XLM_ROBERTA_XL_START_DOCSTRING, + """ ) class XLMRobertaXLForSequenceClassification(XLMRobertaXLPreTrainedModel): def __init__(self, config): @@ -1302,12 +1164,7 @@ class XLMRobertaXLForSequenceClassification(XLMRobertaXLPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1378,13 +1235,7 @@ class XLMRobertaXLForSequenceClassification(XLMRobertaXLPreTrainedModel): ) -@add_start_docstrings( - """ - XLM-RoBERTa-XL Model with a multiple choice classification head on top (a linear layer on top of the pooled - output and a softmax) e.g. for RocStories/SWAG tasks. - """, - XLM_ROBERTA_XL_START_DOCSTRING, -) +@auto_docstring class XLMRobertaXLForMultipleChoice(XLMRobertaXLPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1395,14 +1246,7 @@ class XLMRobertaXLForMultipleChoice(XLMRobertaXLPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward( - XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") - ) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1417,10 +1261,28 @@ class XLMRobertaXLForMultipleChoice(XLMRobertaXLPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input + IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1469,13 +1331,7 @@ class XLMRobertaXLForMultipleChoice(XLMRobertaXLPreTrainedModel): ) -@add_start_docstrings( - """ - XLM-RoBERTa-XL Model with a token classification head on top (a linear layer on top of the hidden-states - output) e.g. for Named-Entity-Recognition (NER) tasks. - """, - XLM_ROBERTA_XL_START_DOCSTRING, -) +@auto_docstring class XLMRobertaXLForTokenClassification(XLMRobertaXLPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1490,12 +1346,7 @@ class XLMRobertaXLForTokenClassification(XLMRobertaXLPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1580,13 +1431,7 @@ class XLMRobertaXLClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - XLM-RoBERTa-XL Model with a span classification head on top for extractive question-answering tasks like SQuAD - (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XLM_ROBERTA_XL_START_DOCSTRING, -) +@auto_docstring class XLMRobertaXLForQuestionAnswering(XLMRobertaXLPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1597,12 +1442,7 @@ class XLMRobertaXLForQuestionAnswering(XLMRobertaXLPreTrainedModel): self.init_weights() - @add_start_docstrings_to_model_forward(XLM_ROBERTA_XL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1617,16 +1457,6 @@ class XLMRobertaXLForQuestionAnswering(XLMRobertaXLPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.roberta( diff --git a/src/transformers/models/xlnet/modeling_xlnet.py b/src/transformers/models/xlnet/modeling_xlnet.py index de7446e57b..5a677f1d71 100755 --- a/src/transformers/models/xlnet/modeling_xlnet.py +++ b/src/transformers/models/xlnet/modeling_xlnet.py @@ -29,22 +29,12 @@ from ...activations import ACT2FN, get_activation from ...generation import GenerationMixin from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_xlnet import XLNetConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "xlnet/xlnet-base-cased" -_CONFIG_FOR_DOC = "XLNetConfig" - def build_tf_xlnet_to_pytorch_map(model, config, tf_weights=None): """ @@ -804,12 +794,8 @@ class XLNetSequenceSummary(nn.Module): return output +@auto_docstring class XLNetPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = XLNetConfig load_tf_weights = load_tf_weights_in_xlnet base_model_prefix = "transformer" @@ -1106,99 +1092,7 @@ class XLNetForQuestionAnsweringOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor, ...]] = None -XLNET_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`XLNetConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -XLNET_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - mems (`List[torch.FloatTensor]` of length `config.n_layers`): - Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential - decoding. The token ids which have their past given to this model should not be passed as `input_ids` as - they have already been computed. - - `use_mems` has to be set to `True` to make use of `mems`. - perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): - Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: - - - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; - - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. - - If not set, each token attends to all the others (full bidirectional attention). Only used during - pretraining (to define factorization order) or for sequential decoding (generation). - target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): - Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is - on the j-th token. Only used during pretraining for partial prediction or for sequential decoding - (generation). - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - input_mask (`torch.FloatTensor` of shape `{0}`, *optional*): - Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for - real tokens and 1 for padding which is kept for compatibility with the original code base. - - Mask values selected in `[0, 1]`: - - - 1 for tokens that are **masked**, - - 0 for tokens that are **not masked**. - - You can only uses one of `input_mask` and `attention_mask`. - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.", - XLNET_START_DOCSTRING, -) +@auto_docstring class XLNetModel(XLNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1327,12 +1221,7 @@ class XLNetModel(XLNetPreTrainedModel): return pos_emb - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XLNetModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1350,6 +1239,40 @@ class XLNetModel(XLNetPreTrainedModel): return_dict: Optional[bool] = None, **kwargs, # delete after depreciation warning is removed ) -> Union[Tuple, XLNetModelOutput]: + r""" + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1556,11 +1479,10 @@ class XLNetModel(XLNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings). - """, - XLNET_START_DOCSTRING, + """ ) class XLNetLMHeadModel(XLNetPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_loss.weight"] @@ -1626,8 +1548,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel, GenerationMixin): return inputs - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=XLNetLMHeadModelOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1647,6 +1568,34 @@ class XLNetLMHeadModel(XLNetPreTrainedModel, GenerationMixin): **kwargs, # delete when `use_cache` is removed in XLNetModel ) -> Union[Tuple, XLNetLMHeadModelOutput]: r""" + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. labels (`torch.LongTensor` of shape `(batch_size, num_predict)`, *optional*): Labels for masked language modeling. `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`. @@ -1657,8 +1606,10 @@ class XLNetLMHeadModel(XLNetPreTrainedModel, GenerationMixin): Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the loss is only computed for labels in `[0, ..., config.vocab_size]` - - Return: + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. Examples: @@ -1763,12 +1714,11 @@ class XLNetLMHeadModel(XLNetPreTrainedModel, GenerationMixin): return [layer_past.index_select(1, beam_idx.to(layer_past.device)) for layer_past in mems] -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - XLNET_START_DOCSTRING, + """ ) class XLNetForSequenceClassification(XLNetPreTrainedModel): def __init__(self, config): @@ -1783,12 +1733,7 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XLNetForSequenceClassificationOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1808,10 +1753,42 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): **kwargs, # delete when `use_cache` is removed in XLNetModel ) -> Union[Tuple, XLNetForSequenceClassificationOutput]: r""" + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1872,13 +1849,7 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): ) -@add_start_docstrings( - """ - XLNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - XLNET_START_DOCSTRING, -) +@auto_docstring class XLNetForTokenClassification(XLNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1890,12 +1861,7 @@ class XLNetForTokenClassification(XLNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XLNetForTokenClassificationOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1915,9 +1881,43 @@ class XLNetForTokenClassification(XLNetPreTrainedModel): **kwargs, # delete when `use_cache` is removed in XLNetModel ) -> Union[Tuple, XLNetForTokenClassificationOutput]: r""" + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above) + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks.emory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1959,13 +1959,7 @@ class XLNetForTokenClassification(XLNetPreTrainedModel): ) -@add_start_docstrings( - """ - XLNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RACE/SWAG tasks. - """, - XLNET_START_DOCSTRING, -) +@auto_docstring class XLNetForMultipleChoice(XLNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1977,12 +1971,7 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XLNetForMultipleChoiceOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2002,10 +1991,59 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): **kwargs, # delete when `use_cache` is removed in XLNetModel ) -> Union[Tuple, XLNetForMultipleChoiceOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + input_mask (`torch.FloatTensor` of shape `batch_size, num_choices, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., - num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See - `input_ids` above) + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2062,12 +2100,11 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): ) -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XLNET_START_DOCSTRING, + """ ) class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): def __init__(self, config): @@ -2080,12 +2117,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=XLNetForQuestionAnsweringSimpleOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2106,14 +2138,38 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): **kwargs, # delete when `use_cache` is removed in XLNetModel ) -> Union[Tuple, XLNetForQuestionAnsweringSimpleOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -2172,13 +2228,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): ) -@add_start_docstrings( - """ - XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XLNET_START_DOCSTRING, -) +@auto_docstring class XLNetForQuestionAnswering(XLNetPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -2193,8 +2243,7 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XLNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @replace_return_docstrings(output_type=XLNetForQuestionAnsweringOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -2218,14 +2267,34 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): **kwargs, # delete when `use_cache` is removed in XLNetModel ) -> Union[Tuple, XLNetForQuestionAnsweringOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + mems (`List[torch.FloatTensor]` of length `config.n_layers`): + Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential + decoding. The token ids which have their past given to this model should not be passed as `input_ids` as + they have already been computed. + + `use_mems` has to be set to `True` to make use of `mems`. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: + + - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; + - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. + + If not set, each token attends to all the others (full bidirectional attention). Only used during + pretraining (to define factorization order) or for sequential decoding (generation). + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is + on the j-th token. Only used during pretraining for partial prediction or for sequential decoding + (generation). + input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*): + Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for + real tokens and 1 for padding which is kept for compatibility with the original code base. + + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **masked**, + - 0 for tokens that are **not masked**. + + You can only uses one of `input_mask` and `attention_mask`. is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels whether a question has an answer or no answer (SQuAD 2.0) cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*): @@ -2234,8 +2303,10 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be masked. 0.0 mean token is not masked. - - Returns: + use_mems (`bool`, *optional*): + Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden + states from previous forward passes to compute attention, which can significantly improve performance for + sequential decoding tasks. Example: diff --git a/src/transformers/models/xmod/modeling_xmod.py b/src/transformers/models/xmod/modeling_xmod.py index 9465bf7a65..646c952fde 100644 --- a/src/transformers/models/xmod/modeling_xmod.py +++ b/src/transformers/models/xmod/modeling_xmod.py @@ -36,7 +36,7 @@ from ...modeling_outputs import ( ) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import auto_docstring, logging from .configuration_xmod import XmodConfig @@ -635,12 +635,8 @@ class XmodPooler(nn.Module): return pooled_output +@auto_docstring class XmodPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = XmodConfig base_model_prefix = "roberta" supports_gradient_checkpointing = True @@ -694,82 +690,8 @@ class XmodPreTrainedModel(PreTrainedModel): parameter.requires_grad = False -XMOD_START_DOCSTRING = r""" - - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`XmodConfig`]): Model configuration class with all the parameters of the - model. Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -XMOD_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - lang_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of the language adapters that should be activated for each sample, respectively. Default: the index - that corresponds to `self.config.default_language`. - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare X-MOD Model transformer outputting raw hidden-states without any specific head on top.", - XMOD_START_DOCSTRING, -) -class XmodModel(XmodPreTrainedModel): - """ - +@auto_docstring( + custom_intro=""" The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz @@ -780,11 +702,15 @@ class XmodModel(XmodPreTrainedModel): `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 - """ - +) +class XmodModel(XmodPreTrainedModel): # Copied from transformers.models.clap.modeling_clap.ClapTextModel.__init__ with ClapText->Xmod def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -813,7 +739,7 @@ class XmodModel(XmodPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -832,25 +758,9 @@ class XmodModel(XmodPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors: - of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). + lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -955,9 +865,10 @@ class XmodModel(XmodPreTrainedModel): ) -@add_start_docstrings( - "X-MOD Model with a `language modeling` head on top for CLM fine-tuning.", - XMOD_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + X-MOD Model with a `language modeling` head on top for CLM fine-tuning. + """ ) class XmodForCausalLM(XmodPreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -983,7 +894,7 @@ class XmodForCausalLM(XmodPreTrainedModel, GenerationMixin): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1004,31 +915,13 @@ class XmodForCausalLM(XmodPreTrainedModel, GenerationMixin): **kwargs, ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - + lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - - Returns: `transformers.modeling_outputs.CausalLMOutputWithCrossAttentions` or `tuple(torch.FloatTensor)` Example: @@ -1103,10 +996,7 @@ class XmodForCausalLM(XmodPreTrainedModel, GenerationMixin): return reordered_past -@add_start_docstrings( - """X-MOD Model with a `language modeling` head on top.""", - XMOD_START_DOCSTRING, -) +@auto_docstring class XmodForMaskedLM(XmodPreTrainedModel): _tied_weights_keys = ["lm_head.decoder.weight", "lm_head.decoder.bias"] @@ -1134,7 +1024,7 @@ class XmodForMaskedLM(XmodPreTrainedModel): def set_output_embeddings(self, new_embeddings): self.lm_head.decoder = new_embeddings - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1152,12 +1042,13 @@ class XmodForMaskedLM(XmodPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" + lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` - kwargs (`Dict[str, any]`, *optional*, defaults to *{}*): - Used to hide legacy arguments that have been deprecated. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1227,12 +1118,11 @@ class XmodLMHead(nn.Module): self.bias = self.decoder.bias -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" X-MOD Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. - """, - XMOD_START_DOCSTRING, + """ ) class XmodForSequenceClassification(XmodPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification.__init__ with Roberta->Xmod @@ -1247,7 +1137,7 @@ class XmodForSequenceClassification(XmodPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1263,6 +1153,9 @@ class XmodForSequenceClassification(XmodPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" + lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If @@ -1320,13 +1213,7 @@ class XmodForSequenceClassification(XmodPreTrainedModel): ) -@add_start_docstrings( - """ - X-MOD Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a - softmax) e.g. for RocStories/SWAG tasks. - """, - XMOD_START_DOCSTRING, -) +@auto_docstring class XmodForMultipleChoice(XmodPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForMultipleChoice.__init__ with Roberta->Xmod def __init__(self, config): @@ -1339,7 +1226,7 @@ class XmodForMultipleChoice(XmodPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1355,10 +1242,37 @@ class XmodForMultipleChoice(XmodPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + lang_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] @@ -1409,13 +1323,7 @@ class XmodForMultipleChoice(XmodPreTrainedModel): ) -@add_start_docstrings( - """ - X-MOD Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for - Named-Entity-Recognition (NER) tasks. - """, - XMOD_START_DOCSTRING, -) +@auto_docstring class XmodForTokenClassification(XmodPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForTokenClassification.__init__ with Roberta->Xmod def __init__(self, config): @@ -1432,7 +1340,7 @@ class XmodForTokenClassification(XmodPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1448,6 +1356,9 @@ class XmodForTokenClassification(XmodPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" + lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ @@ -1511,13 +1422,7 @@ class XmodClassificationHead(nn.Module): return x -@add_start_docstrings( - """ - X-MOD Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - XMOD_START_DOCSTRING, -) +@auto_docstring class XmodForQuestionAnswering(XmodPreTrainedModel): # Copied from transformers.models.roberta.modeling_roberta.RobertaForQuestionAnswering.__init__ with Roberta->Xmod def __init__(self, config): @@ -1530,7 +1435,7 @@ class XmodForQuestionAnswering(XmodPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(XMOD_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1547,14 +1452,9 @@ class XmodForQuestionAnswering(XmodPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. + lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of the language adapters that should be activated for each sample, respectively. Default: the index + that corresponds to `self.config.default_language`. """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/yolos/image_processing_yolos_fast.py b/src/transformers/models/yolos/image_processing_yolos_fast.py index d048b3bb0e..cd29e647a0 100644 --- a/src/transformers/models/yolos/image_processing_yolos_fast.py +++ b/src/transformers/models/yolos/image_processing_yolos_fast.py @@ -9,8 +9,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union from ...image_processing_utils import BatchFeature, get_size_dict from ...image_processing_utils_fast import ( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, BaseImageProcessorFast, DefaultFastImageProcessorKwargs, SizeDict, @@ -33,7 +31,7 @@ from ...image_utils import ( from ...processing_utils import Unpack from ...utils import ( TensorType, - add_start_docstrings, + auto_docstring, is_torch_available, is_torchvision_available, is_torchvision_v2_available, @@ -58,6 +56,26 @@ logger = logging.get_logger(__name__) class YolosFastImageProcessorKwargs(DefaultFastImageProcessorKwargs): + r""" + format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): + Data format of the annotations. One of "coco_detection" or "coco_panoptic". + do_convert_annotations (`bool`, *optional*, defaults to `True`): + Controls whether to convert the annotations to the format expected by the YOLOS model. Converts the + bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. + Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` + method. If `True`, padding will be applied to the bottom and right of the image with zeros. + If `pad_size` is provided, the image will be padded to the specified dimensions. + Otherwise, the image will be padded to the maximum height and width of the batch. + pad_size (`Dict[str, int]`, *optional*): + The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size + provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest + height and width in the batch. + return_segmentation_masks (`bool`, *optional*, defaults to `False`): + Whether to return segmentation masks. + """ + format: Optional[Union[str, AnnotationFormat]] do_convert_annotations: Optional[bool] do_pad: Optional[bool] @@ -314,29 +332,7 @@ def get_size_with_aspect_ratio( return (oh, ow) -@add_start_docstrings( - "Constructs a fast Yolos image processor.", - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING, - """ - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the YOLOS model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. - """, -) +@auto_docstring @requires(backends=("torchvision", "torch")) class YolosImageProcessorFast(BaseImageProcessorFast): resample = PILImageResampling.BILINEAR @@ -610,9 +606,15 @@ class YolosImageProcessorFast(BaseImageProcessorFast): return image, pixel_mask, annotation - @add_start_docstrings( - BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS, - """ + @auto_docstring + def preprocess( + self, + images: ImageInput, + annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, + masks_path: Optional[Union[str, pathlib.Path]] = None, + **kwargs: Unpack[YolosFastImageProcessorKwargs], + ) -> BatchFeature: + r""" annotations (`AnnotationType` or `List[AnnotationType]`, *optional*): List of annotations associated with the image or batch of images. If annotation is for object detection, the annotations should be a dictionary with the following keys: @@ -626,32 +628,9 @@ class YolosImageProcessorFast(BaseImageProcessorFast): - "file_name" (`str`): The file name of the image. format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the YOLOS model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Controls whether to pad the image. Can be overridden by the `do_pad` parameter in the `preprocess` - method. If `True`, padding will be applied to the bottom and right of the image with zeros. - If `pad_size` is provided, the image will be padded to the specified dimensions. - Otherwise, the image will be padded to the maximum height and width of the batch. - pad_size (`Dict[str, int]`, *optional*): - The size `{"height": int, "width" int}` to pad the images to. Must be larger than any image size - provided for preprocessing. If `pad_size` is not provided, images will be padded to the largest - height and width in the batch. - return_segmentation_masks (`bool`, *optional*, defaults to `False`): - Whether to return segmentation masks. masks_path (`str` or `pathlib.Path`, *optional*): Path to the directory containing the segmentation masks. - """, - ) - def preprocess( - self, - images: ImageInput, - annotations: Optional[Union[AnnotationType, List[AnnotationType]]] = None, - masks_path: Optional[Union[str, pathlib.Path]] = None, - **kwargs: Unpack[YolosFastImageProcessorKwargs], - ) -> BatchFeature: + """ if "pad_and_return_pixel_mask" in kwargs: kwargs["do_pad"] = kwargs.pop("pad_and_return_pixel_mask") logger.warning_once( diff --git a/src/transformers/models/yolos/modeling_yolos.py b/src/transformers/models/yolos/modeling_yolos.py index 2705a412b7..2443591f63 100755 --- a/src/transformers/models/yolos/modeling_yolos.py +++ b/src/transformers/models/yolos/modeling_yolos.py @@ -26,26 +26,12 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import ModelOutput, auto_docstring, logging from .configuration_yolos import YolosConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "YolosConfig" - -# Base docstring -_CHECKPOINT_FOR_DOC = "hustvl/yolos-small" -_EXPECTED_OUTPUT_SHAPE = [1, 3401, 384] - @dataclass class YolosObjectDetectionOutput(ModelOutput): @@ -537,12 +523,8 @@ class YolosEncoder(nn.Module): ) +@auto_docstring class YolosPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = YolosConfig base_model_prefix = "vit" main_input_name = "pixel_values" @@ -564,46 +546,13 @@ class YolosPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -YOLOS_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`YolosConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -YOLOS_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See - [`YolosImageProcessor.__call__`] for details. - - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare YOLOS Model transformer outputting raw hidden-states without any specific head on top.", - YOLOS_START_DOCSTRING, -) +@auto_docstring class YolosModel(YolosPreTrainedModel): def __init__(self, config: YolosConfig, add_pooling_layer: bool = True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ super().__init__(config) self.config = config @@ -631,14 +580,7 @@ class YolosModel(YolosPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(YOLOS_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPooling, - config_class=_CONFIG_FOR_DOC, - modality="vision", - expected_output=_EXPECTED_OUTPUT_SHAPE, - ) + @auto_docstring def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -727,11 +669,10 @@ class YolosMLPPredictionHead(nn.Module): return x -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" YOLOS Model (consisting of a ViT encoder) with object detection heads on top, for tasks such as COCO detection. - """, - YOLOS_START_DOCSTRING, + """ ) class YolosForObjectDetection(YolosPreTrainedModel): def __init__(self, config: YolosConfig): @@ -760,8 +701,7 @@ class YolosForObjectDetection(YolosPreTrainedModel): # as a dict having both a Tensor and a list. return [{"logits": a, "pred_boxes": b} for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] - @add_start_docstrings_to_model_forward(YOLOS_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=YolosObjectDetectionOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -778,8 +718,6 @@ class YolosForObjectDetection(YolosPreTrainedModel): boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`. - Returns: - Examples: ```python diff --git a/src/transformers/models/yoso/modeling_yoso.py b/src/transformers/models/yoso/modeling_yoso.py index edccabee2e..9bebac2303 100644 --- a/src/transformers/models/yoso/modeling_yoso.py +++ b/src/transformers/models/yoso/modeling_yoso.py @@ -35,9 +35,7 @@ from ...modeling_outputs import ( from ...modeling_utils import PreTrainedModel from ...pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, + auto_docstring, is_ninja_available, is_torch_cuda_available, logging, @@ -47,9 +45,6 @@ from .configuration_yoso import YosoConfig logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "uw-madison/yoso-4096" -_CONFIG_FOR_DOC = "YosoConfig" - lsh_cumulation = None @@ -643,12 +638,8 @@ class YosoOnlyMLMHead(nn.Module): return prediction_scores +@auto_docstring class YosoPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = YosoConfig base_model_prefix = "yoso" supports_gradient_checkpointing = True @@ -670,71 +661,7 @@ class YosoPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -YOSO_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use - it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`YosoConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -YOSO_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, - 1]`: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - [What are token type IDs?](../glossary#token-type-ids) - position_ids (`torch.LongTensor` of shape `({0})`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - - [What are position IDs?](../glossary#position-ids) - head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): - Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert *input_ids* indices into associated vectors than the - model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - "The bare YOSO Model transformer outputting raw hidden-states without any specific head on top.", - YOSO_START_DOCSTRING, -) +@auto_docstring class YosoModel(YosoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -760,12 +687,7 @@ class YosoModel(YosoPreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithCrossAttentions, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -842,7 +764,7 @@ class YosoModel(YosoPreTrainedModel): ) -@add_start_docstrings("""YOSO Model with a `language modeling` head on top.""", YOSO_START_DOCSTRING) +@auto_docstring class YosoForMaskedLM(YosoPreTrainedModel): _tied_weights_keys = ["cls.predictions.decoder.weight", "cls.predictions.decoder.bias"] @@ -862,12 +784,7 @@ class YosoForMaskedLM(YosoPreTrainedModel): self.cls.predictions.decoder = new_embeddings self.cls.predictions.bias = new_embeddings.bias - @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MaskedLMOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -942,10 +859,11 @@ class YosoClassificationHead(nn.Module): return x -@add_start_docstrings( - """YOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of - the pooled output) e.g. for GLUE tasks.""", - YOSO_START_DOCSTRING, +@auto_docstring( + custom_intro=""" + YOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. + """ ) class YosoForSequenceClassification(YosoPreTrainedModel): def __init__(self, config): @@ -957,12 +875,7 @@ class YosoForSequenceClassification(YosoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=SequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1033,11 +946,7 @@ class YosoForSequenceClassification(YosoPreTrainedModel): ) -@add_start_docstrings( - """YOSO Model with a multiple choice classification head on top (a linear layer on top of - the pooled output and a softmax) e.g. for RocStories/SWAG tasks.""", - YOSO_START_DOCSTRING, -) +@auto_docstring class YosoForMultipleChoice(YosoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1049,12 +958,7 @@ class YosoForMultipleChoice(YosoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=MultipleChoiceModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1069,6 +973,30 @@ class YosoForMultipleChoice(YosoPreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, MultipleChoiceModelOutput]: r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert *input_ids* indices into associated vectors than the + model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See @@ -1124,11 +1052,7 @@ class YosoForMultipleChoice(YosoPreTrainedModel): ) -@add_start_docstrings( - """YOSO Model with a token classification head on top (a linear layer on top of - the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.""", - YOSO_START_DOCSTRING, -) +@auto_docstring class YosoForTokenClassification(YosoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1141,12 +1065,7 @@ class YosoForTokenClassification(YosoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TokenClassifierOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1209,11 +1128,7 @@ class YosoForTokenClassification(YosoPreTrainedModel): ) -@add_start_docstrings( - """YOSO Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layers on top of the hidden-states output to compute `span start logits` and `span end logits`).""", - YOSO_START_DOCSTRING, -) +@auto_docstring class YosoForQuestionAnswering(YosoPreTrainedModel): def __init__(self, config): super().__init__(config) @@ -1227,12 +1142,7 @@ class YosoForQuestionAnswering(YosoPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(YOSO_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=QuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - ) + @auto_docstring def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1247,16 +1157,6 @@ class YosoForQuestionAnswering(YosoPreTrainedModel): output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, QuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence - are not taken into account for computing the loss. - """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.yoso( diff --git a/src/transformers/models/zamba/modeling_zamba.py b/src/transformers/models/zamba/modeling_zamba.py index 31949cf6a0..8e5d1f6567 100644 --- a/src/transformers/models/zamba/modeling_zamba.py +++ b/src/transformers/models/zamba/modeling_zamba.py @@ -30,28 +30,14 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin -from ...modeling_attn_mask_utils import ( - AttentionMaskConverter, -) +from ...modeling_attn_mask_utils import AttentionMaskConverter from ...modeling_flash_attention_utils import FlashAttentionKwargs -from ...modeling_outputs import ( - BaseModelOutputWithPast, - CausalLMOutputWithPast, - SequenceClassifierOutputWithPast, -) +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack from ...pytorch_utils import ALL_LAYERNORM_LAYERS -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) -from ...utils.import_utils import ( - is_causal_conv1d_available, - is_mamba_ssm_available, -) +from ...utils import auto_docstring, logging +from ...utils.import_utils import is_causal_conv1d_available, is_mamba_ssm_available from .configuration_zamba import ZambaConfig @@ -73,8 +59,6 @@ is_fast_path_available = all( logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "ZambaConfig" - # Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Zamba class ZambaRMSNorm(nn.Module): @@ -807,27 +791,7 @@ class ZambaHybridLayer(nn.Module): return layer_outputs -ZAMBA_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ZambaConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Zamba Model outputting raw hidden-states without any specific head on top.", - ZAMBA_START_DOCSTRING, -) +@auto_docstring class ZambaPreTrainedModel(PreTrainedModel): config_class = ZambaConfig base_model_prefix = "model" @@ -896,79 +860,7 @@ class ZambaPreTrainedModel(PreTrainedModel): return config -ZAMBA_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`ZambaHybridDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - A ZambaHybridDynamicCache object containing pre-computed hidden-states (keys and values in the - self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`. - Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and - `(batch_size, d_inner, d_state)` respectively. - See the `ZambaHybridDynamicCache` class for more details. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Zamba Model outputting raw hidden-states without any specific head on top.", - ZAMBA_START_DOCSTRING, -) +@auto_docstring class ZambaModel(ZambaPreTrainedModel): """ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ZambaDecoderLayer`] @@ -1030,7 +922,7 @@ class ZambaModel(ZambaPreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(ZAMBA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1207,8 +1099,7 @@ class ZambaForCausalLM(ZambaPreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(ZAMBA_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1226,19 +1117,10 @@ class ZambaForCausalLM(ZambaPreTrainedModel, GenerationMixin): **loss_kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1359,8 +1241,8 @@ class ZambaForCausalLM(ZambaPreTrainedModel, GenerationMixin): return model_inputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Zamba Model with a sequence classification head on top (linear layer). [`ZambaForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1371,8 +1253,7 @@ class ZambaForCausalLM(ZambaPreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - ZAMBA_START_DOCSTRING, + """ ) class ZambaForSequenceClassification(ZambaPreTrainedModel): def __init__(self, config): @@ -1391,7 +1272,7 @@ class ZambaForSequenceClassification(ZambaPreTrainedModel): def set_input_embeddings(self, value): self.model.embed_tokens = value - @add_start_docstrings_to_model_forward(ZAMBA_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/zamba2/modeling_zamba2.py b/src/transformers/models/zamba2/modeling_zamba2.py index 19fc05b217..5f2c382c67 100644 --- a/src/transformers/models/zamba2/modeling_zamba2.py +++ b/src/transformers/models/zamba2/modeling_zamba2.py @@ -37,12 +37,7 @@ from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from ...processing_utils import Unpack -from ...utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import auto_docstring, logging from ...utils.import_utils import is_causal_conv1d_available, is_mamba_ssm_available from .configuration_zamba2 import Zamba2Config @@ -62,9 +57,6 @@ else: logger = logging.get_logger(__name__) -_CONFIG_FOR_DOC = "Zyphra/Zamba2-2.7B" - - class Zamba2RMSNormGated(torch.nn.Module): def __init__(self, hidden_size, group_size, eps=1e-6): super().__init__() @@ -1241,96 +1233,7 @@ class Zamba2PreTrainedModel(PreTrainedModel): module.D.data.fill_(1.0) -ZAMBA2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Zamba2Config`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -ZAMBA2_INPUTS_DOCSTRING = r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - If `past_key_values` is used, optionally only the last `input_ids` have to be input (see - `past_key_values`). - - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] - and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more - information on the default strategy. - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.n_positions - 1]`. - - [What are position IDs?](../glossary#position-ids) - past_key_values (`Zamba2HybridDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - A Zamba2HybridDynamicCache object containing pre-computed hidden-states (keys and values in the - self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see - `past_key_values` input) to speed up sequential decoding. - Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`. - Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and - `(batch_size, d_inner, d_state)` respectively. - See the `Zamba2HybridDynamicCache` class for more details. - - If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This - is useful if you want more control over how to convert `input_ids` indices into associated vectors than the - model's internal embedding lookup matrix. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, - this tensor is not affected by padding. It is used to update the cache in the correct position and to infer - the complete sequence length. -""" - - -@add_start_docstrings( - "The bare Zamba2 Model outputting raw hidden-states without any specific head on top.", - ZAMBA2_START_DOCSTRING, -) +@auto_docstring class Zamba2Model(Zamba2PreTrainedModel): """ Model consisting of *config.num_hidden_layers* layers. @@ -1381,7 +1284,7 @@ class Zamba2Model(Zamba2PreTrainedModel): def set_input_embeddings(self, value): self.embed_tokens = value - @add_start_docstrings_to_model_forward(ZAMBA2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1616,8 +1519,7 @@ class Zamba2ForCausalLM(Zamba2PreTrainedModel, GenerationMixin): def get_decoder(self): return self.model - @add_start_docstrings_to_model_forward(ZAMBA2_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1635,19 +1537,10 @@ class Zamba2ForCausalLM(Zamba2PreTrainedModel, GenerationMixin): **loss_kwargs, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - logits_to_keep (`int` or `torch.Tensor`, *optional*): - If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all - `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that - token can save memory, which becomes pretty significant for long sequences or large vocabulary size. - If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. - This is useful when using packed tensor format (single dimension for batch and sequence length). - - Returns: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: @@ -1768,8 +1661,8 @@ class Zamba2ForCausalLM(Zamba2PreTrainedModel, GenerationMixin): return model_inputs -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" The Zamba2 Model with a sequence classification head on top (linear layer). [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models @@ -1780,8 +1673,7 @@ class Zamba2ForCausalLM(Zamba2PreTrainedModel, GenerationMixin): no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). - """, - ZAMBA2_START_DOCSTRING, + """ ) class Zamba2ForSequenceClassification(Zamba2PreTrainedModel): def __init__(self, config): @@ -1800,7 +1692,7 @@ class Zamba2ForSequenceClassification(Zamba2PreTrainedModel): def set_input_embeddings(self, value): self.model.embed_tokens = value - @add_start_docstrings_to_model_forward(ZAMBA2_INPUTS_DOCSTRING) + @auto_docstring def forward( self, input_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/zoedepth/modeling_zoedepth.py b/src/transformers/models/zoedepth/modeling_zoedepth.py index 5f2e1a8656..5084ffff35 100644 --- a/src/transformers/models/zoedepth/modeling_zoedepth.py +++ b/src/transformers/models/zoedepth/modeling_zoedepth.py @@ -23,23 +23,15 @@ import torch.utils.checkpoint from torch import nn from ...activations import ACT2FN -from ...file_utils import ( - add_start_docstrings, - add_start_docstrings_to_model_forward, - replace_return_docstrings, -) from ...modeling_outputs import DepthEstimatorOutput from ...modeling_utils import PreTrainedModel -from ...utils import ModelOutput, logging +from ...utils import ModelOutput, auto_docstring, logging from ...utils.backbone_utils import load_backbone from .configuration_zoedepth import ZoeDepthConfig logger = logging.get_logger(__name__) -# General docstring -_CONFIG_FOR_DOC = "ZoeDepthConfig" - @dataclass class ZoeDepthDepthEstimatorOutput(ModelOutput): @@ -1221,12 +1213,8 @@ class ZoeDepthMetricDepthEstimationHead(nn.Module): # Modified from transformers.models.dpt.modeling_dpt.DPTPreTrainedModel with DPT->ZoeDepth,dpt->zoedepth # avoiding sdpa and flash_attn_2 support, it's done int the backend +@auto_docstring class ZoeDepthPreTrainedModel(PreTrainedModel): - """ - An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained - models. - """ - config_class = ZoeDepthConfig base_model_prefix = "zoedepth" main_input_name = "pixel_values" @@ -1245,39 +1233,10 @@ class ZoeDepthPreTrainedModel(PreTrainedModel): module.weight.data.fill_(1.0) -ZOEDEPTH_START_DOCSTRING = r""" - This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it - as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and - behavior. - - Parameters: - config ([`ViTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - -ZOEDEPTH_INPUTS_DOCSTRING = r""" - Args: - pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`DPTImageProcessor.__call__`] - for details. - - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. -""" - - -@add_start_docstrings( - """ +@auto_docstring( + custom_intro=""" ZoeDepth model with one or multiple metric depth estimation head(s) on top. - """, - ZOEDEPTH_START_DOCSTRING, + """ ) class ZoeDepthForDepthEstimation(ZoeDepthPreTrainedModel): def __init__(self, config): @@ -1305,8 +1264,7 @@ class ZoeDepthForDepthEstimation(ZoeDepthPreTrainedModel): # Initialize weights and apply final processing self.post_init() - @add_start_docstrings_to_model_forward(ZOEDEPTH_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=DepthEstimatorOutput, config_class=_CONFIG_FOR_DOC) + @auto_docstring def forward( self, pixel_values: torch.FloatTensor, @@ -1319,8 +1277,6 @@ class ZoeDepthForDepthEstimation(ZoeDepthPreTrainedModel): labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth depth estimation maps for computing the loss. - Returns: - Examples: ```python >>> from transformers import AutoImageProcessor, ZoeDepthForDepthEstimation diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 50b1a57c3b..bb2f7e890f 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -21,6 +21,17 @@ from huggingface_hub.constants import HF_HUB_DISABLE_TELEMETRY as DISABLE_TELEME from packaging import version from .. import __version__ +from .args_doc import ( + ClassAttrs, + ClassDocstring, + ImageProcessorArgs, + ModelArgs, + auto_class_docstring, + auto_docstring, + parse_docstring, + set_min_indent, + source_args_doc, +) from .backbone_utils import BackboneConfigMixin, BackboneMixin from .chat_template_utils import DocstringParsingException, TypeHintParsingException, get_json_schema from .constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD diff --git a/src/transformers/utils/args_doc.py b/src/transformers/utils/args_doc.py new file mode 100644 index 0000000000..7048a223bb --- /dev/null +++ b/src/transformers/utils/args_doc.py @@ -0,0 +1,1572 @@ +# coding=utf-8 +# Copyright 2025 HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os +import textwrap +from pathlib import Path +from typing import List, Optional, Tuple, Union, get_args + +import regex as re + +from .doc import ( + MODELS_TO_PIPELINE, + PIPELINE_TASKS_TO_SAMPLE_DOCSTRINGS, + PT_SAMPLE_DOCSTRINGS, + _prepare_output_docstrings, +) +from .generic import ModelOutput + + +PATH_TO_TRANSFORMERS = Path("src").resolve() / "transformers" + + +AUTODOC_FILES = [ + "configuration_*.py", + "modeling_*.py", + "tokenization_*.py", + "processing_*.py", + "image_processing_*_fast.py", + "image_processing_*.py", + "feature_extractor_*.py", +] + +PLACEHOLDER_TO_AUTO_MODULE = { + "image_processor_class": ("image_processing_auto", "IMAGE_PROCESSOR_MAPPING_NAMES"), + "feature_extractor_class": ("feature_extraction_auto", "FEATURE_EXTRACTOR_MAPPING_NAMES"), + "processor_class": ("processing_auto", "PROCESSOR_MAPPING_NAMES"), + "config_class": ("configuration_auto", "CONFIG_MAPPING_NAMES"), +} + +UNROLL_KWARGS_METHODS = { + "preprocess", +} + +UNROLL_KWARGS_CLASSES = { + "ImageProcessorFast", +} + +HARDCODED_CONFIG_FOR_MODELS = { + "openai": "OpenAIGPTConfig", + "x-clip": "XCLIPConfig", + "kosmos2": "Kosmos2Config", + "donut": "DonutSwinConfig", + "esmfold": "EsmConfig", +} + +_re_checkpoint = re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)") + + +class ImageProcessorArgs: + images = { + "description": """ + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + """, + "shape": None, + } + + videos = { + "description": """ + Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If + passing in videos with pixel values between 0 and 1, set `do_rescale=False`. + """, + "shape": None, + } + + do_resize = { + "description": """ + Whether to resize the image. + """, + "shape": None, + } + + size = { + "description": """ + Describes the maximum input dimensions to the model. + """, + "shape": None, + } + + default_to_square = { + "description": """ + Whether to default to a square image when resizing, if size is an int. + """, + "shape": None, + } + + resample = { + "description": """ + Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only + has an effect if `do_resize` is set to `True`. + """, + "shape": None, + } + + do_center_crop = { + "description": """ + Whether to center crop the image. + """, + "shape": None, + } + + crop_size = { + "description": """ + Size of the output image after applying `center_crop`. + """, + "shape": None, + } + + do_rescale = { + "description": """ + Whether to rescale the image. + """, + "shape": None, + } + + rescale_factor = { + "description": """ + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + """, + "shape": None, + } + + do_normalize = { + "description": """ + Whether to normalize the image. + """, + "shape": None, + } + + image_mean = { + "description": """ + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. + """, + "shape": None, + } + + image_std = { + "description": """ + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to + `True`. + """, + "shape": None, + } + + do_convert_rgb = { + "description": """ + Whether to convert the image to RGB. + """, + "shape": None, + } + + return_tensors = { + "description": """ + Returns stacked tensors if set to `pt, otherwise returns a list of tensors. + """, + "shape": None, + } + + data_format = { + "description": """ + Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors. + """, + "shape": None, + } + + input_data_format = { + "description": """ + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + """, + "shape": None, + } + + device = { + "description": """ + The device to process the images on. If unset, the device is inferred from the input images. + """, + "shape": None, + } + + +class ModelArgs: + labels = { + "description": """ + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + num_logits_to_keep = { + "description": """ + Calculate logits for the last `num_logits_to_keep` tokens. If `0`, calculate logits for all + `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that + token can save memory, which becomes pretty significant for long sequences or large vocabulary size. + """, + "shape": None, + } + + input_ids = { + "description": """ + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + input_values = { + "description": """ + Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file + into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install + soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and + conversion into a tensor of type `torch.FloatTensor`. See [`{processor_class}.__call__`] for details. + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + attention_mask = { + "description": """ + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + head_mask = { + "description": """ + Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """, + "shape": "of shape `(num_heads,)` or `(num_layers, num_heads)`", + } + + cross_attn_head_mask = { + "description": """ + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """, + "shape": "of shape `(num_layers, num_heads)`", + } + + decoder_attention_mask = { + "description": """ + Mask to avoid performing attention on certain token indices. By default, a causal mask will be used, to + make sure the model can only look at previous inputs in order to predict the future. + """, + "shape": "of shape `(batch_size, target_sequence_length)`", + } + + decoder_head_mask = { + "description": """ + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + """, + "shape": "of shape `(decoder_layers, decoder_attention_heads)`", + } + + encoder_hidden_states = { + "description": """ + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + if the model is configured as a decoder. + """, + "shape": "of shape `(batch_size, sequence_length, hidden_size)`", + } + + encoder_attention_mask = { + "description": """ + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + token_type_ids = { + "description": """ + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + position_ids = { + "description": """ + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + """, + "shape": "of shape `(batch_size, sequence_length)`", + } + + past_key_values = { + "description": """ + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + Two formats are allowed: + - a [`~cache_utils.Cache`] instance, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); + - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy + cache format. + + The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the + legacy cache format will be returned. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + """, + "shape": None, + } + + past_key_value = { + "description": """ + deprecated in favor of `past_key_values` + """, + "shape": None, + } + + inputs_embeds = { + "description": """ + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + """, + "shape": "of shape `(batch_size, sequence_length, hidden_size)`", + } + + decoder_input_ids = { + "description": """ + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + """, + "shape": "of shape `(batch_size, target_sequence_length)`", + } + + decoder_inputs_embeds = { + "description": """ + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value + of `inputs_embeds`. + """, + "shape": "of shape `(batch_size, target_sequence_length, hidden_size)`", + } + + use_cache = { + "description": """ + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + """, + "shape": None, + } + + output_attentions = { + "description": """ + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + """, + "shape": None, + } + + output_hidden_states = { + "description": """ + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + """, + "shape": None, + } + + return_dict = { + "description": """ + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """, + "shape": None, + } + + cache_position = { + "description": """ + Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, + this tensor is not affected by padding. It is used to update the cache in the correct position and to infer + the complete sequence length. + """, + "shape": "of shape `(sequence_length)`", + } + + hidden_states = { + "description": """ input to the layer of shape `(batch, seq_len, embed_dim)""", + "shape": None, + } + + interpolate_pos_encoding = { + "description": """ + Whether to interpolate the pre-trained position encodings. + """, + "shape": None, + } + + position_embeddings = { + "description": """ + Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, + with `head_dim` being the embedding dimension of each attention head. + """, + "shape": None, + } + + config = { + "description": """ + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. + """, + "shape": None, + } + + start_positions = { + "description": """ + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + """, + "shape": "of shape `(batch_size,)`", + } + + end_positions = { + "description": """ + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + """, + "shape": "of shape `(batch_size,)`", + } + + encoder_outputs = { + "description": """ + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + """, + "shape": None, + } + + output_router_logits = { + "description": """ + Whether or not to return the logits of all the routers. They are useful for computing the router loss, and + should not be returned during inference. + """, + "shape": None, + } + + logits_to_keep = { + "description": """ + If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all + `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that + token can save memory, which becomes pretty significant for long sequences or large vocabulary size. + If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. + This is useful when using packed tensor format (single dimension for batch and sequence length). + """, + "shape": None, + } + + pixel_values = { + "description": """ + The tensors corresponding to the input images. Pixel values can be obtained using + [`{image_processor_class}`]. See [`{image_processor_class}.__call__`] for details ([`{processor_class}`] uses + [`{image_processor_class}`] for processing images). + """, + "shape": "of shape `(batch_size, num_channels, image_size, image_size)`", + } + + vision_feature_layer = { + "description": """ + The index of the layer to select the vision feature. If multiple indices are provided, + the vision feature of the corresponding indices will be concatenated to form the + vision features. + """, + "shape": None, + } + + vision_feature_select_strategy = { + "description": """ + The feature selection strategy used to select the vision feature from the vision backbone. + Can be one of `"default"` or `"full"`. + """, + "shape": None, + } + + image_sizes = { + "description": """ + The sizes of the images in the batch, being (height, width) for each image. + """, + "shape": "of shape `(batch_size, 2)`", + } + + pixel_mask = { + "description": """ + Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: + + - 1 for pixels that are real (i.e. **not masked**), + - 0 for pixels that are padding (i.e. **masked**). + + [What are attention masks?](../glossary#attention-mask) + """, + "shape": "of shape `(batch_size, height, width)`", + } + + +class ClassDocstring: + PreTrainedModel = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + """ + + Model = r""" + The bare {model_name} Model outputting raw hidden-states without any specific head on top. + """ + + ForPreTraining = r""" + The {model_name} Model with a specified pretraining head on top. + """ + + Decoder = r""" + The bare {model_name} Decoder outputting raw hidden-states without any specific head on top. + """ + + TextModel = r""" + The bare {model_name} Text Model outputting raw hidden-states without any specific head on to. + """ + + ForSequenceClassification = r""" + The {model_name} Model with a sequence classification/regression head on top e.g. for GLUE tasks. + """ + + ForQuestionAnswering = r""" + The {model_name} transformer with a span classification head on top for extractive question-answering tasks like + SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + """ + + ForMultipleChoice = r""" + The {model_name} Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a + softmax) e.g. for RocStories/SWAG tasks. + """ + + ForMaskedLM = r""" + The {model_name} Model with a `language modeling` head on top." + """ + + ForTokenClassification = r""" + The {model_name} transformer with a token classification head on top (a linear layer on top of the hidden-states + output) e.g. for Named-Entity-Recognition (NER) tasks. + """ + + ForConditionalGeneration = r""" + The {model_name} Model for token generation conditioned on other modalities (e.g. image-text-to-text generation). + """ + + ForCausalLM = r""" + The {model_name} Model for causal language modeling. + """ + + ImageProcessorFast = r""" + Constructs a fast {model_name} image processor. + """ + + Backbone = r""" + The {model_name} backbone. + """ + + ForImageClassification = r""" + The {model_name} Model with an image classification head on top e.g. for ImageNet. + """ + ForSemanticSegmentation = r""" + The {model_name} Model with a semantic segmentation head on top e.g. for ADE20K, CityScapes. + """ + ForAudioClassification = r""" + The {model_name} Model with an audio classification head on top (a linear layer on top of the pooled + output). + """ + + ForAudioFrameClassification = r""" + The {model_name} Model with a frame classification head on top for tasks like Speaker Diarization. + """ + + ForPrediction = r""" + The {model_name} Model with a distribution head on top for time-series forecasting. + """ + + WithProjection = r""" + The {model_name} Model with a projection layer on top (a linear layer on top of the pooled output). + """ + + +class ClassAttrs: + # fmt: off + base_model_prefix = r""" + A string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. + """ + supports_gradient_checkpointing = r""" + Whether the model supports gradient checkpointing or not. Gradient checkpointing is a memory-saving technique that trades compute for memory, by storing only a subset of activations (checkpoints) and recomputing the activations that are not stored during the backward pass. + """ + _no_split_modules = r""" + Layers of modules that should not be split across devices should be added to `_no_split_modules`. This can be useful for modules that contains skip connections or other operations that are not compatible with splitting the module across devices. Setting this attribute will enable the use of `device_map="auto"` in the `from_pretrained` method. + """ + _skip_keys_device_placement = r""" + A list of keys to ignore when moving inputs or outputs between devices when using the `accelerate` library. + """ + _supports_flash_attn_2 = r""" + Whether the model's attention implementation supports FlashAttention 2.0. + """ + _supports_sdpa = r""" + Whether the model's attention implementation supports SDPA (Scaled Dot Product Attention). + """ + _supports_flex_attn = r""" + Whether the model's attention implementation supports FlexAttention. + """ + _supports_cache_class = r""" + Whether the model supports a `Cache` instance as `past_key_values`. + """ + _supports_quantized_cache = r""" + Whether the model supports a `QuantoQuantizedCache` instance as `past_key_values`. + """ + _supports_static_cache = r""" + Whether the model supports a `StaticCache` instance as `past_key_values`. + """ + _supports_attention_backend = r""" + Whether the model supports attention interface functions. This flag signal that the model can be used as an efficient backend in TGI and vLLM. + """ + _tied_weights_keys = r""" + A list of `state_dict` keys that are potentially tied to another key in the state_dict. + """ + # fmt: on + + +ARGS_TO_IGNORE = {"self", "kwargs", "args", "deprecated_arguments"} + + +def get_indent_level(func): + # Use this instead of `inspect.getsource(func)` as getsource can be very slow + return (len(func.__qualname__.split(".")) - 1) * 4 + + +def equalize_indent(docstring, indent_level): + """ + Adjust the indentation of a docstring to match the specified indent level. + """ + # fully dedent the docstring + docstring = "\n".join([line.lstrip() for line in docstring.splitlines()]) + return textwrap.indent(docstring, " " * indent_level) + + +def set_min_indent(docstring, indent_level): + """ + Adjust the indentation of a docstring to match the specified indent level. + """ + return textwrap.indent(textwrap.dedent(docstring), " " * indent_level) + + +def parse_shape(docstring): + shape_pattern = re.compile(r"(of shape\s*(?:`.*?`|\(.*?\)))") + match = shape_pattern.search(docstring) + if match: + return " " + match.group(1) + return None + + +def parse_default(docstring): + default_pattern = re.compile(r"(defaults to \s*[^)]*)") + match = default_pattern.search(docstring) + if match: + return " " + match.group(1) + return None + + +def parse_docstring(docstring, max_indent_level=0): + """ + Parse the docstring to extract the Args section and return it as a dictionary. + The docstring is expected to be in the format: + Args: + arg1 (type): Description of arg1. + arg2 (type): Description of arg2. + + # This function will also return the remaining part of the docstring after the Args section. + Returns:/Example: + ... + """ + match = re.search(r"(?m)^([ \t]*)(?=Example|Return)", docstring) + if match: + remainder_docstring = docstring[match.start() :] + docstring = docstring[: match.start()] + else: + remainder_docstring = "" + args_pattern = re.compile(r"(?:Args:)(\n.*)?(\n)?$", re.DOTALL) + + args_match = args_pattern.search(docstring) + # still try to find args description in the docstring, if args are not preceded by "Args:" + args_section = args_match.group(1).lstrip("\n") if args_match else docstring + if args_section.split("\n")[-1].strip() == '"""': + args_section = "\n".join(args_section.split("\n")[:-1]) + if args_section.split("\n")[0].strip() == 'r"""' or args_section.split("\n")[0].strip() == '"""': + args_section = "\n".join(args_section.split("\n")[1:]) + args_section = set_min_indent(args_section, 0) + + params = {} + if args_section: + param_pattern = re.compile( + # |--- Group 1 ---|| Group 2 ||- Group 3 -||---------- Group 4 ----------| + rf"^\s{{0,{max_indent_level}}}(\w+)\s*\(\s*([^, \)]*)(\s*.*?)\s*\)\s*:\s*((?:(?!\n^\s{{0,{max_indent_level}}}\w+\s*\().)*)", + re.DOTALL | re.MULTILINE, + ) + for match in param_pattern.finditer(args_section): + param_name = match.group(1) + param_type = match.group(2) + # param_type = match.group(2).replace("`", "") + additional_info = match.group(3) + optional = "optional" in additional_info + shape = parse_shape(additional_info) + default = parse_default(additional_info) + param_description = match.group(4).strip() + # set first line of param_description to 4 spaces: + param_description = re.sub(r"^", " " * 4, param_description, 1) + param_description = f"\n{param_description}" + params[param_name] = { + "type": param_type, + "description": param_description, + "optional": optional, + "shape": shape, + "default": default, + "additional_info": additional_info, + } + + if params and remainder_docstring: + remainder_docstring = "\n" + remainder_docstring + + remainder_docstring = set_min_indent(remainder_docstring, 0) + + return params, remainder_docstring + + +def contains_type(type_hint, target_type) -> Tuple[bool, Optional[object]]: + """ + Check if a "nested" type hint contains a specific target type, + return the first-level type containing the target_type if found. + """ + args = get_args(type_hint) + if args == (): + try: + return issubclass(type_hint, target_type), type_hint + except Exception as _: + return issubclass(type(type_hint), target_type), type_hint + found_type_tuple = [contains_type(arg, target_type)[0] for arg in args] + found_type = any(found_type_tuple) + if found_type: + type_hint = args[found_type_tuple.index(True)] + return found_type, type_hint + + +def get_model_name(obj): + """ + Get the model name from the file path of the object. + """ + path = inspect.getsourcefile(obj) + if path.split(os.path.sep)[-3] != "models": + return None + file_name = path.split(os.path.sep)[-1] + for file_type in AUTODOC_FILES: + start = file_type.split("*")[0] + end = file_type.split("*")[-1] if "*" in file_type else "" + if file_name.startswith(start) and file_name.endswith(end): + model_name_lowercase = file_name[len(start) : -len(end)] + return model_name_lowercase + else: + print(f"🚨 Something went wrong trying to find the model name in the path: {path}") + return "model" + + +def get_placeholders_dict(placeholders: List, model_name: str) -> dict: + """ + Get the dictionary of placeholders for the given model name. + """ + # import here to avoid circular import + from transformers.models import auto as auto_module + + placeholders_dict = {} + for placeholder in placeholders: + # Infer placeholders from the model name and the auto modules + if placeholder in PLACEHOLDER_TO_AUTO_MODULE: + place_holder_value = getattr( + getattr(auto_module, PLACEHOLDER_TO_AUTO_MODULE[placeholder][0]), + PLACEHOLDER_TO_AUTO_MODULE[placeholder][1], + )[model_name] + if isinstance(place_holder_value, (list, tuple)): + place_holder_value = place_holder_value[0] + placeholders_dict[placeholder] = place_holder_value + + return placeholders_dict + + +def format_args_docstring(args, model_name): + """ + Replaces placeholders such as {image_processor_class} in the docstring with the actual values, + deducted from the model name and the auto modules. + """ + # first check if there are any placeholders in the args, if not return them as is + placeholders = set(re.findall(r"{(.*?)}", "".join((args[arg]["description"] for arg in args)))) + if not placeholders: + return args + + # get the placeholders dictionary for the given model name + placeholders_dict = get_placeholders_dict(placeholders, model_name) + + # replace the placeholders in the args with the values from the placeholders_dict + for arg in args: + new_arg = args[arg]["description"] + placeholders = re.findall(r"{(.*?)}", new_arg) + placeholders = [placeholder for placeholder in placeholders if placeholder in placeholders_dict] + if placeholders: + new_arg = new_arg.format(**{placeholder: placeholders_dict[placeholder] for placeholder in placeholders}) + args[arg]["description"] = new_arg + + return args + + +def source_args_doc(args_classes: Union[object, List[object]]) -> dict: + if isinstance(args_classes, (list, tuple)): + args_classes_dict = {} + for args_class in args_classes: + args_classes_dict.update(args_class.__dict__) + return args_classes_dict + return args_classes.__dict__ + + +def get_checkpoint_from_config_class(config_class): + checkpoint = None + + # source code of `config_class` + # config_source = inspect.getsource(config_class) + config_source = config_class.__doc__ + checkpoints = _re_checkpoint.findall(config_source) + # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link. + # For example, `('google-bert/bert-base-uncased', 'https://huggingface.co/google-bert/bert-base-uncased')` + for ckpt_name, ckpt_link in checkpoints: + # allow the link to end with `/` + if ckpt_link.endswith("/"): + ckpt_link = ckpt_link[:-1] + + # verify the checkpoint name corresponds to the checkpoint link + ckpt_link_from_name = f"https://huggingface.co/{ckpt_name}" + if ckpt_link == ckpt_link_from_name: + checkpoint = ckpt_name + break + + return checkpoint + + +def add_intro_docstring(func, class_name, parent_class=None, indent_level=0): + intro_docstring = "" + if func.__name__ == "forward": + intro_docstring = rf"""The [`{class_name}`] forward method, overrides the `__call__` special method. + + + + Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`] + instance afterwards instead of this since the former takes care of running the pre and post processing steps while + the latter silently ignores them. + + + + """ + intro_docstring = equalize_indent(intro_docstring, indent_level + 4) + + return intro_docstring + + +def _get_model_info(func, parent_class): + """ + Extract model information from a function or its parent class. + + Args: + func (`function`): The function to extract information from + parent_class (`class`): Optional parent class of the function + """ + # import here to avoid circular import + from transformers.models import auto as auto_module + + # Get model name from either parent class or function + if parent_class is not None: + model_name_lowercase = get_model_name(parent_class) + else: + model_name_lowercase = get_model_name(func) + + # Normalize model name if needed + if model_name_lowercase and model_name_lowercase not in getattr( + getattr(auto_module, PLACEHOLDER_TO_AUTO_MODULE["config_class"][0]), + PLACEHOLDER_TO_AUTO_MODULE["config_class"][1], + ): + model_name_lowercase = model_name_lowercase.replace("_", "-") + + # Get class name from function's qualified name + class_name = func.__qualname__.split(".")[0] + + # Get config class for the model + if model_name_lowercase is None: + config_class = None + else: + try: + config_class = getattr( + getattr(auto_module, PLACEHOLDER_TO_AUTO_MODULE["config_class"][0]), + PLACEHOLDER_TO_AUTO_MODULE["config_class"][1], + )[model_name_lowercase] + except KeyError: + if model_name_lowercase in HARDCODED_CONFIG_FOR_MODELS: + config_class = HARDCODED_CONFIG_FOR_MODELS[model_name_lowercase] + else: + config_class = "ModelConfig" + print( + f"🚨 Config not found for {model_name_lowercase}. You can manually add it to HARDCODED_CONFIG_FOR_MODELS in utils/args_doc.py" + ) + + return model_name_lowercase, class_name, config_class + + +def _process_parameter_type(param, param_name, func): + """ + Process and format a parameter's type annotation. + + Args: + param (`inspect.Parameter`): The parameter from the function signature + param_name (`str`): The name of the parameter + func (`function`): The function the parameter belongs to + """ + optional = False + if param.annotation != inspect.Parameter.empty: + param_type = param.annotation + if "typing" in str(param_type): + param_type = "".join(str(param_type).split("typing.")).replace("transformers.", "~") + elif hasattr(param_type, "__module__"): + param_type = f"{param_type.__module__.replace('transformers.', '~').replace('builtins', '')}.{param.annotation.__name__}" + if param_type[0] == ".": + param_type = param_type[1:] + else: + if False: + print( + f"🚨 {param_type} for {param_name} of {func.__qualname__} in file {func.__code__.co_filename} has an invalid type" + ) + if "ForwardRef" in param_type: + param_type = re.sub(r"ForwardRef\('([\w.]+)'\)", r"\1", param_type) + if "Optional" in param_type: + param_type = re.sub(r"Optional\[(.*?)\]", r"\1", param_type) + optional = True + else: + param_type = "" + + return param_type, optional + + +def _get_parameter_info(param_name, documented_params, source_args_dict, param_type, optional): + """ + Get parameter documentation details from the appropriate source. + Tensor shape, optional status and description are taken from the custom docstring in priority if available. + Type is taken from the function signature first, then from the custom docstring if missing from the signature + + Args: + param_name (`str`): Name of the parameter + documented_params (`dict`): Dictionary of documented parameters (manually specified in the docstring) + source_args_dict (`dict`): Default source args dictionary to use if not in documented_params + param_type (`str`): Current parameter type (may be updated) + optional (`bool`): Whether the parameter is optional (may be updated) + """ + description = None + shape = None + shape_string = "" + is_documented = True + additional_info = None + + if param_name in documented_params: + # Parameter is documented in the function's docstring + if param_type == "" and documented_params[param_name].get("type", None) is not None: + param_type = documented_params[param_name]["type"] + optional = documented_params[param_name]["optional"] + shape = documented_params[param_name]["shape"] + shape_string = shape if shape else "" + additional_info = documented_params[param_name]["additional_info"] or "" + description = f"{documented_params[param_name]['description']}\n" + elif param_name in source_args_dict: + # Parameter is documented in ModelArgs or ImageProcessorArgs + shape = source_args_dict[param_name]["shape"] + shape_string = " " + shape if shape else "" + description = source_args_dict[param_name]["description"] + additional_info = None + else: + # Parameter is not documented + is_documented = False + optional_string = r", *optional*" if optional else "" + + return param_type, optional_string, shape_string, additional_info, description, is_documented + + +def _process_regular_parameters(sig, func, class_name, documented_params, indent_level, undocumented_parameters): + """ + Process all regular parameters (not kwargs parameters) from the function signature. + + Args: + sig (`inspect.Signature`): Function signature + func (`function`): Function the parameters belong to + class_name (`str`): Name of the class + documented_params (`dict`): Dictionary of parameters that are already documented + indent_level (`int`): Indentation level + undocumented_parameters (`list`): List to append undocumented parameters to + """ + docstring = "" + source_args_dict = source_args_doc([ModelArgs, ImageProcessorArgs]) + missing_args = {} + + for param_name, param in sig.parameters.items(): + # Skip parameters that should be ignored + if ( + param_name in ARGS_TO_IGNORE + or param.kind == inspect.Parameter.VAR_POSITIONAL + or param.kind == inspect.Parameter.VAR_KEYWORD + ): + continue + + # Process parameter type and optional status + param_type, optional = _process_parameter_type(param, param_name, func) + + # Check for default value + param_default = "" + if param.default != inspect._empty and param.default is not None: + param_default = f", defaults to `{str(param.default)}`" + + param_type, optional_string, shape_string, additional_info, description, is_documented = _get_parameter_info( + param_name, documented_params, source_args_dict, param_type, optional + ) + + if is_documented: + if param_name == "config": + if param_type == "": + param_type = f"[`{class_name}`]" + else: + param_type = f"[`{param_type.split('.')[-1]}`]" + elif param_type == "" and False: # TODO: Enforce typing for all parameters + print(f"🚨 {param_name} for {func.__qualname__} in file {func.__code__.co_filename} has no type") + param_type = param_type if "`" in param_type else f"`{param_type}`" + # Format the parameter docstring + if additional_info: + param_docstring = f"{param_name} ({param_type}{additional_info}):{description}" + else: + param_docstring = ( + f"{param_name} ({param_type}{shape_string}{optional_string}{param_default}):{description}" + ) + docstring += set_min_indent( + param_docstring, + indent_level + 8, + ) + else: + missing_args[param_name] = { + "type": param_type if param_type else "", + "optional": optional, + "shape": shape_string, + "description": description if description else "\n ", + "default": param_default, + } + undocumented_parameters.append( + f"🚨 `{param_name}` is part of {func.__qualname__}'s signature, but not documented. Make sure to add it to the docstring of the function in {func.__code__.co_filename}." + ) + + return docstring, missing_args + + +def find_sig_line(lines, line_end): + parenthesis_count = 0 + sig_line_end = line_end + found_sig = False + while not found_sig: + for char in lines[sig_line_end]: + if char == "(": + parenthesis_count += 1 + elif char == ")": + parenthesis_count -= 1 + if parenthesis_count == 0: + found_sig = True + break + sig_line_end += 1 + return sig_line_end + + +def _process_kwargs_parameters( + sig, func, parent_class, model_name_lowercase, documented_kwargs, indent_level, undocumented_parameters +): + """ + Process **kwargs parameters if needed. + + Args: + sig (`inspect.Signature`): Function signature + func (`function`): Function the parameters belong to + parent_class (`class`): Parent class of the function + model_name_lowercase (`str`): Lowercase model name + documented_kwargs (`dict`): Dictionary of kwargs that are already documented + indent_level (`int`): Indentation level + undocumented_parameters (`list`): List to append undocumented parameters to + """ + docstring = "" + source_args_dict = source_args_doc(ImageProcessorArgs) + + # Check if we need to add typed kwargs description to the docstring + unroll_kwargs = func.__name__ in UNROLL_KWARGS_METHODS + if not unroll_kwargs and parent_class is not None: + # Check if the function has a parent class with unroll kwargs + unroll_kwargs = any( + unroll_kwargs_class in parent_class.__name__ for unroll_kwargs_class in UNROLL_KWARGS_CLASSES + ) + + if unroll_kwargs: + # get all unpackable "kwargs" parameters + kwargs_parameters = [ + kwargs_param + for _, kwargs_param in sig.parameters.items() + if kwargs_param.kind == inspect.Parameter.VAR_KEYWORD + ] + for kwarg_param in kwargs_parameters: + # If kwargs not typed, skip + if kwarg_param.annotation == inspect.Parameter.empty: + continue + + # Extract documentation for kwargs + kwargs_documentation = kwarg_param.annotation.__args__[0].__doc__ + if kwargs_documentation is not None: + documented_kwargs, _ = parse_docstring(kwargs_documentation) + if model_name_lowercase is not None: + documented_kwargs = format_args_docstring(documented_kwargs, model_name_lowercase) + + # Process each kwarg parameter + for param_name, param_type_annotation in kwarg_param.annotation.__args__[0].__annotations__.items(): + param_type = str(param_type_annotation) + optional = False + + # Process parameter type + if "typing" in param_type: + param_type = "".join(param_type.split("typing.")).replace("transformers.", "~") + else: + param_type = f"{param_type.replace('transformers.', '~').replace('builtins', '')}.{param_name}" + if "ForwardRef" in param_type: + param_type = re.sub(r"ForwardRef\('([\w.]+)'\)", r"\1", param_type) + if "Optional" in param_type: + param_type = re.sub(r"Optional\[(.*?)\]", r"\1", param_type) + optional = True + + # Check for default value + param_default = "" + if parent_class is not None: + param_default = str(getattr(parent_class, param_name, "")) + param_default = f", defaults to `{param_default}`" if param_default != "" else "" + + param_type, optional_string, shape_string, additional_info, description, is_documented = ( + _get_parameter_info(param_name, documented_kwargs, source_args_dict, param_type, optional) + ) + + if is_documented: + # Check if type is missing + if param_type == "": + print( + f"🚨 {param_name} for {kwarg_param.annotation.__args__[0].__qualname__} in file {func.__code__.co_filename} has no type" + ) + param_type = param_type if "`" in param_type else f"`{param_type}`" + # Format the parameter docstring + if additional_info: + docstring += set_min_indent( + f"{param_name} ({param_type}{additional_info}):{description}", + indent_level + 8, + ) + else: + docstring += set_min_indent( + f"{param_name} ({param_type}{shape_string}{optional_string}{param_default}):{description}", + indent_level + 8, + ) + else: + undocumented_parameters.append( + f"🚨 `{param_name}` is part of {kwarg_param.annotation.__args__[0].__qualname__}, but not documented. Make sure to add it to the docstring of the function in {func.__code__.co_filename}." + ) + + return docstring + + +def _process_parameters_section( + func_documentation, sig, func, class_name, model_name_lowercase, parent_class, indent_level +): + """ + Process the parameters section of the docstring. + + Args: + func_documentation (`str`): Existing function documentation (manually specified in the docstring) + sig (`inspect.Signature`): Function signature + func (`function`): Function the parameters belong to + class_name (`str`): Name of the class the function belongs to + model_name_lowercase (`str`): Lowercase model name + parent_class (`class`): Parent class of the function (if any) + indent_level (`int`): Indentation level + """ + # Start Args section + docstring = set_min_indent("Args:\n", indent_level + 4) + undocumented_parameters = [] + documented_params = {} + documented_kwargs = {} + + # Parse existing docstring if available + if func_documentation is not None: + documented_params, func_documentation = parse_docstring(func_documentation) + if model_name_lowercase is not None: + documented_params = format_args_docstring(documented_params, model_name_lowercase) + + # Process regular parameters + param_docstring, missing_args = _process_regular_parameters( + sig, func, class_name, documented_params, indent_level, undocumented_parameters + ) + docstring += param_docstring + + # Process **kwargs parameters if needed + kwargs_docstring = _process_kwargs_parameters( + sig, func, parent_class, model_name_lowercase, documented_kwargs, indent_level, undocumented_parameters + ) + docstring += kwargs_docstring + + # Report undocumented parameters + if len(undocumented_parameters) > 0: + print("\n".join(undocumented_parameters)) + + return docstring + + +def _process_returns_section(func_documentation, sig, config_class, indent_level): + """ + Process the returns section of the docstring. + + Args: + func_documentation (`str`): Existing function documentation (manually specified in the docstring) + sig (`inspect.Signature`): Function signature + config_class (`str`): Config class for the model + indent_level (`int`): Indentation level + """ + return_docstring = "" + + # Extract returns section from existing docstring if available + if ( + func_documentation is not None + and (match_start := re.search(r"(?m)^([ \t]*)(?=Return)", func_documentation)) is not None + ): + match_end = re.search(r"(?m)^([ \t]*)(?=Example)", func_documentation) + if match_end: + return_docstring = func_documentation[match_start.start() : match_end.start()] + func_documentation = func_documentation[match_end.start() :] + else: + return_docstring = func_documentation[match_start.start() :] + func_documentation = "" + return_docstring = set_min_indent(return_docstring, indent_level + 4) + # Otherwise, generate return docstring from return annotation if available + elif sig.return_annotation is not None and sig.return_annotation != inspect._empty: + add_intro, return_annotation = contains_type(sig.return_annotation, ModelOutput) + return_docstring = _prepare_output_docstrings(return_annotation, config_class, add_intro=add_intro) + return_docstring = return_docstring.replace("typing.", "") + return_docstring = set_min_indent(return_docstring, indent_level + 4) + + return return_docstring, func_documentation + + +def _process_example_section( + func_documentation, func, parent_class, class_name, model_name_lowercase, config_class, checkpoint, indent_level +): + """ + Process the example section of the docstring. + + Args: + func_documentation (`str`): Existing function documentation (manually specified in the docstring) + func (`function`): Function being processed + parent_class (`class`): Parent class of the function + class_name (`str`): Name of the class + model_name_lowercase (`str`): Lowercase model name + config_class (`str`): Config class for the model + checkpoint: Checkpoint to use in examples + indent_level (`int`): Indentation level + """ + # Import here to avoid circular import + from transformers.models import auto as auto_module + + example_docstring = "" + + # Use existing example section if available + + if func_documentation is not None and (match := re.search(r"(?m)^([ \t]*)(?=Example)", func_documentation)): + example_docstring = func_documentation[match.start() :] + example_docstring = "\n" + set_min_indent(example_docstring, indent_level + 4) + # No examples for __init__ methods or if the class is not a model + elif parent_class is None and model_name_lowercase is not None: + task = rf"({'|'.join(PT_SAMPLE_DOCSTRINGS.keys())})" + model_task = re.search(task, class_name) + CONFIG_MAPPING = auto_module.configuration_auto.CONFIG_MAPPING + + # Get checkpoint example + if (checkpoint_example := checkpoint) is None: + try: + checkpoint_example = get_checkpoint_from_config_class(CONFIG_MAPPING[model_name_lowercase]) + except KeyError: + # For models with inconsistent lowercase model name + if model_name_lowercase in HARDCODED_CONFIG_FOR_MODELS: + CONFIG_MAPPING_NAMES = auto_module.configuration_auto.CONFIG_MAPPING_NAMES + config_class_name = HARDCODED_CONFIG_FOR_MODELS[model_name_lowercase] + if config_class_name in CONFIG_MAPPING_NAMES.values(): + model_name_for_auto_config = [ + k for k, v in CONFIG_MAPPING_NAMES.items() if v == config_class_name + ][0] + if model_name_for_auto_config in CONFIG_MAPPING: + checkpoint_example = get_checkpoint_from_config_class( + CONFIG_MAPPING[model_name_for_auto_config] + ) + + # Add example based on model task + if model_task is not None: + if checkpoint_example is not None: + example_annotation = "" + task = model_task.group() + example_annotation = PT_SAMPLE_DOCSTRINGS[task].format( + model_class=class_name, + checkpoint=checkpoint_example, + expected_output="...", + expected_loss="...", + qa_target_start_index=14, + qa_target_end_index=15, + mask="", + ) + example_docstring = set_min_indent(example_annotation, indent_level + 4) + else: + print( + f"🚨 No checkpoint found for {class_name}.{func.__name__}. Please add a `checkpoint` arg to `auto_docstring` or add one in {config_class}'s docstring" + ) + else: + # Check if the model is in a pipeline to get an example + for name_model_list_for_task in MODELS_TO_PIPELINE: + model_list_for_task = getattr(auto_module.modeling_auto, name_model_list_for_task) + if class_name in model_list_for_task.values(): + pipeline_name = MODELS_TO_PIPELINE[name_model_list_for_task] + example_annotation = PIPELINE_TASKS_TO_SAMPLE_DOCSTRINGS[pipeline_name].format( + model_class=class_name, + checkpoint=checkpoint_example, + expected_output="...", + expected_loss="...", + qa_target_start_index=14, + qa_target_end_index=15, + ) + example_docstring = set_min_indent(example_annotation, indent_level + 4) + break + + return example_docstring + + +def auto_method_docstring(func, parent_class=None, custom_intro=None, custom_args=None, checkpoint=None): + """ + Wrapper that automatically generates docstring. + """ + + # Use inspect to retrieve the method's signature + sig = inspect.signature(func) + indent_level = get_indent_level(func) + + # Get model information + model_name_lowercase, class_name, config_class = _get_model_info(func, parent_class) + func_documentation = func.__doc__ + if custom_args is not None and func_documentation is not None: + func_documentation = set_min_indent(custom_args, indent_level + 4) + "\n" + func_documentation + elif custom_args is not None: + func_documentation = custom_args + + # Add intro to the docstring before args description if needed + if custom_intro is not None: + docstring = set_min_indent(custom_intro, indent_level + 4) + else: + docstring = add_intro_docstring( + func, class_name=class_name, parent_class=parent_class, indent_level=indent_level + ) + + # Process Parameters section + docstring += _process_parameters_section( + func_documentation, sig, func, class_name, model_name_lowercase, parent_class, indent_level + ) + + # Process Returns section + return_docstring, func_documentation = _process_returns_section( + func_documentation, sig, config_class, indent_level + ) + docstring += return_docstring + + # Process Example section + example_docstring = _process_example_section( + func_documentation, + func, + parent_class, + class_name, + model_name_lowercase, + config_class, + checkpoint, + indent_level, + ) + docstring += example_docstring + + # Assign the dynamically generated docstring to the wrapper function + func.__doc__ = docstring + return func + + +def auto_class_docstring(cls, custom_intro=None, custom_args=None, checkpoint=None): + """ + Wrapper that automatically generates a docstring for classes based on their attributes and methods. + """ + # import here to avoid circular import + from transformers.models import auto as auto_module + + docstring_init = auto_method_docstring(cls.__init__, parent_class=cls, custom_args=custom_args).__doc__.replace( + "Args:", "Parameters:" + ) + indent_level = get_indent_level(cls) + model_name_lowercase = get_model_name(cls) + model_name_title = " ".join([k.title() for k in model_name_lowercase.split("_")]) if model_name_lowercase else None + if model_name_lowercase and model_name_lowercase not in getattr( + getattr(auto_module, PLACEHOLDER_TO_AUTO_MODULE["config_class"][0]), + PLACEHOLDER_TO_AUTO_MODULE["config_class"][1], + ): + model_name_lowercase = model_name_lowercase.replace("_", "-") + + name = re.findall(rf"({'|'.join(ClassDocstring.__dict__.keys())})$", cls.__name__) + if name == [] and cls.__doc__ is None and custom_intro is None: + raise ValueError( + f"`{cls.__name__}` is not part of the auto doc. Here are the available classes: {ClassDocstring.__dict__.keys()}" + ) + if name != [] or custom_intro is not None: + name = name[0] if name else None + if custom_intro is not None: + pre_block = equalize_indent(custom_intro, indent_level) + if not pre_block.endswith("\n"): + pre_block += "\n" + elif model_name_title is None: + pre_block = "" + else: + pre_block = getattr(ClassDocstring, name).format(model_name=model_name_title) + # Start building the docstring + docstring = set_min_indent(f"{pre_block}", indent_level) if len(pre_block) else "" + if name != "PreTrainedModel" and "PreTrainedModel" in (x.__name__ for x in cls.__mro__): + docstring += set_min_indent(f"{ClassDocstring.PreTrainedModel}", indent_level) + # Add the __init__ docstring + docstring += set_min_indent(f"\n{docstring_init}", indent_level) + attr_docs = "" + # Get all attributes and methods of the class + + for attr_name, attr_value in cls.__dict__.items(): + if not callable(attr_value) and not attr_name.startswith("__"): + if attr_value.__class__.__name__ == "property": + attr_type = "property" + else: + attr_type = type(attr_value).__name__ + if name and "Config" in name: + raise ValueError("Config should have explicit docstring") + indented_doc = getattr(ClassAttrs, attr_name, None) + if indented_doc is not None: + attr_docs += set_min_indent(f"{attr_name} (`{attr_type}`): {indented_doc}", 0) + + # TODO: Add support for Attributes section in docs + # if len(attr_docs.replace(" ", "")): + # docstring += set_min_indent("\nAttributes:\n", indent_level) + # docstring += set_min_indent(attr_docs, indent_level + 4) + else: + print( + f"You used `@auto_class_docstring` decorator on `{cls.__name__}` but this class is not part of the AutoMappings. Remove the decorator" + ) + # Assign the dynamically generated docstring to the wrapper class + cls.__doc__ = docstring + + return cls + + +def auto_docstring(obj=None, *, custom_intro=None, custom_args=None, checkpoint=None): + """ + Automatically generates docstrings for classes and methods in the Transformers library. + + This decorator can be used in the following forms: + @auto_docstring + def my_function(...): + ... + or + @auto_docstring() + def my_function(...): + ... + or + @auto_docstring(custom_intro="Custom intro", ...) + def my_function(...): + ... + + Args: + custom_intro (str, optional): Custom introduction text to add to the docstring. This will replace the default + introduction text generated by the decorator before the Args section. + checkpoint (str, optional): Checkpoint name to use in the docstring. This should be automatically inferred from the + model configuration class, but can be overridden if needed. + """ + + def auto_docstring_decorator(obj): + if len(obj.__qualname__.split(".")) > 1: + return auto_method_docstring( + obj, custom_args=custom_args, custom_intro=custom_intro, checkpoint=checkpoint + ) + else: + return auto_class_docstring(obj, custom_args=custom_args, custom_intro=custom_intro, checkpoint=checkpoint) + + if obj: + return auto_docstring_decorator(obj) + + return auto_docstring_decorator diff --git a/src/transformers/utils/doc.py b/src/transformers/utils/doc.py index f01ffc2844..8624aebdce 100644 --- a/src/transformers/utils/doc.py +++ b/src/transformers/utils/doc.py @@ -20,6 +20,7 @@ import inspect import re import textwrap import types +from collections import OrderedDict def get_docstring_indentation_level(func): @@ -135,31 +136,41 @@ def _convert_output_args_doc(output_args_doc): return "\n".join(blocks) -def _prepare_output_docstrings(output_type, config_class, min_indent=None): +def _prepare_output_docstrings(output_type, config_class, min_indent=None, add_intro=True): """ Prepares the return part of the docstring using `output_type`. """ output_docstring = output_type.__doc__ - - # Remove the head of the docstring to keep the list of args only - lines = output_docstring.split("\n") - i = 0 - while i < len(lines) and re.search(r"^\s*(Args|Parameters):\s*$", lines[i]) is None: - i += 1 - if i < len(lines): - params_docstring = "\n".join(lines[(i + 1) :]) - params_docstring = _convert_output_args_doc(params_docstring) - else: - raise ValueError( - f"No `Args` or `Parameters` section is found in the docstring of `{output_type.__name__}`. Make sure it has " - "docstring and contain either `Args` or `Parameters`." - ) + params_docstring = None + if output_docstring is not None: + # Remove the head of the docstring to keep the list of args only + lines = output_docstring.split("\n") + i = 0 + while i < len(lines) and re.search(r"^\s*(Args|Parameters):\s*$", lines[i]) is None: + i += 1 + if i < len(lines): + params_docstring = "\n".join(lines[(i + 1) :]) + params_docstring = _convert_output_args_doc(params_docstring) + elif add_intro: + raise ValueError( + f"No `Args` or `Parameters` section is found in the docstring of `{output_type.__name__}`. Make sure it has " + "docstring and contain either `Args` or `Parameters`." + ) # Add the return introduction - full_output_type = f"{output_type.__module__}.{output_type.__name__}" - intro = TF_RETURN_INTRODUCTION if output_type.__name__.startswith("TF") else PT_RETURN_INTRODUCTION - intro = intro.format(full_output_type=full_output_type, config_class=config_class) - result = intro + params_docstring + if add_intro: + full_output_type = f"{output_type.__module__}.{output_type.__name__}" + intro = TF_RETURN_INTRODUCTION if output_type.__name__.startswith("TF") else PT_RETURN_INTRODUCTION + intro = intro.format(full_output_type=full_output_type, config_class=config_class) + else: + full_output_type = str(output_type) + intro = f"\nReturns:\n `{full_output_type}`" + if params_docstring is not None: + intro += ":\n" + + result = intro + if params_docstring is not None: + result += params_docstring # Apply minimum indent if necessary if min_indent is not None: @@ -635,6 +646,359 @@ PT_SAMPLE_DOCSTRINGS = { } +TEXT_TO_AUDIO_SPECTROGRAM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoProcessor, {model_class}, SpeechT5HifiGan + + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") + >>> inputs = processor(text="Hello, my dog is cute", return_tensors="pt") + + >>> # generate speech + >>> speech = model.generate(inputs["input_ids"], speaker_embeddings=speaker_embeddings, vocoder=vocoder) + ``` +""" + + +TEXT_TO_AUDIO_WAVEFORM_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoProcessor, {model_class} + + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> inputs = processor(text="Hello, my dog is cute", return_tensors="pt") + + >>> # generate speech + >>> speech = model(inputs["input_ids"]) + ``` +""" + + +AUDIO_FRAME_CLASSIFICATION_SAMPLE = PT_SPEECH_FRAME_CLASS_SAMPLE + + +AUDIO_XVECTOR_SAMPLE = PT_SPEECH_XVECTOR_SAMPLE + + +IMAGE_TO_TEXT_SAMPLE = r""" + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, {model_class} + + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + ``` +""" + + +DEPTH_ESTIMATION_SAMPLE = r""" + Example: + + ```python + >>> from transformers import AutoImageProcessor, {model_class} + >>> import torch + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> processor = AutoImageProcessor.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> model.to(device) + + >>> # prepare image for the model + >>> inputs = processor(images=image, return_tensors="pt").to(device) + + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> # interpolate to original size + >>> post_processed_output = processor.post_process_depth_estimation( + ... outputs, [(image.height, image.width)], + ... ) + >>> predicted_depth = post_processed_output[0]["predicted_depth"] + ``` +""" + + +VIDEO_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +ZERO_SHOT_OBJECT_DETECTION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +IMAGE_TO_IMAGE_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +IMAGE_FEATURE_EXTRACTION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +DOCUMENT_QUESTION_ANSWERING_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +NEXT_SENTENCE_PREDICTION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +MULTIPLE_CHOICE_SAMPLE = PT_MULTIPLE_CHOICE_SAMPLE + + +PRETRAINING_SAMPLE = r""" + Example: + + ```python + ``` +""" +MASK_GENERATION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +VISUAL_QUESTION_ANSWERING_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +TEXT_GENERATION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +IMAGE_CLASSIFICATION_SAMPLE = PT_VISION_SEQ_CLASS_SAMPLE + + +IMAGE_SEGMENTATION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +FILL_MASK_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +OBJECT_DETECTION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +QUESTION_ANSWERING_SAMPLE = PT_QUESTION_ANSWERING_SAMPLE + + +TEXT2TEXT_GENERATION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +TEXT_CLASSIFICATION_SAMPLE = PT_SEQUENCE_CLASSIFICATION_SAMPLE + + +TABLE_QUESTION_ANSWERING_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +TOKEN_CLASSIFICATION_SAMPLE = PT_TOKEN_CLASSIFICATION_SAMPLE + + +AUDIO_CLASSIFICATION_SAMPLE = PT_SPEECH_SEQ_CLASS_SAMPLE + + +AUTOMATIC_SPEECH_RECOGNITION_SAMPLE = PT_SPEECH_CTC_SAMPLE + + +ZERO_SHOT_IMAGE_CLASSIFICATION_SAMPLE = r""" + Example: + + ```python + ``` +""" + + +IMAGE_TEXT_TO_TEXT_GENERATION_SAMPLE = r""" + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, {model_class} + + >>> model = {model_class}.from_pretrained("{checkpoint}") + >>> processor = AutoProcessor.from_pretrained("{checkpoint}") + + >>> messages = [ + ... {{ + ... "role": "user", "content": [ + ... {{"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"}}, + ... {{"type": "text", "text": "Where is the cat standing?"}}, + ... ] + ... }}, + ... ] + + >>> inputs = processor.apply_chat_template( + ... messages, + ... tokenizer=True, + ... return_dict=True, + ... return_tensors="pt", + ... add_generation_prompt=True + ... ) + >>> # Generate + >>> generate_ids = model.generate(**inputs) + >>> processor.batch_decode(generate_ids, skip_special_tokens=True)[0] + ``` +""" + + +PIPELINE_TASKS_TO_SAMPLE_DOCSTRINGS = OrderedDict( + [ + ("text-to-audio-spectrogram", TEXT_TO_AUDIO_SPECTROGRAM_SAMPLE), + ("text-to-audio-waveform", TEXT_TO_AUDIO_WAVEFORM_SAMPLE), + ("automatic-speech-recognition", AUTOMATIC_SPEECH_RECOGNITION_SAMPLE), + ("audio-frame-classification", AUDIO_FRAME_CLASSIFICATION_SAMPLE), + ("audio-classification", AUDIO_CLASSIFICATION_SAMPLE), + ("audio-xvector", AUDIO_XVECTOR_SAMPLE), + ("image-text-to-text", IMAGE_TEXT_TO_TEXT_GENERATION_SAMPLE), + ("image-to-text", IMAGE_TO_TEXT_SAMPLE), + ("visual-question-answering", VISUAL_QUESTION_ANSWERING_SAMPLE), + ("depth-estimation", DEPTH_ESTIMATION_SAMPLE), + ("video-classification", VIDEO_CLASSIFICATION_SAMPLE), + ("zero-shot-image-classification", ZERO_SHOT_IMAGE_CLASSIFICATION_SAMPLE), + ("image-classification", IMAGE_CLASSIFICATION_SAMPLE), + ("zero-shot-object-detection", ZERO_SHOT_OBJECT_DETECTION_SAMPLE), + ("object-detection", OBJECT_DETECTION_SAMPLE), + ("image-segmentation", IMAGE_SEGMENTATION_SAMPLE), + ("image-to-image", IMAGE_TO_IMAGE_SAMPLE), + ("image-feature-extraction", IMAGE_FEATURE_EXTRACTION_SAMPLE), + ("text-generation", TEXT_GENERATION_SAMPLE), + ("table-question-answering", TABLE_QUESTION_ANSWERING_SAMPLE), + ("document-question-answering", DOCUMENT_QUESTION_ANSWERING_SAMPLE), + ("question-answering", QUESTION_ANSWERING_SAMPLE), + ("text2text-generation", TEXT2TEXT_GENERATION_SAMPLE), + ("next-sentence-prediction", NEXT_SENTENCE_PREDICTION_SAMPLE), + ("multiple-choice", MULTIPLE_CHOICE_SAMPLE), + ("text-classification", TEXT_CLASSIFICATION_SAMPLE), + ("token-classification", TOKEN_CLASSIFICATION_SAMPLE), + ("fill-mask", FILL_MASK_SAMPLE), + ("mask-generation", MASK_GENERATION_SAMPLE), + ("pretraining", PRETRAINING_SAMPLE), + ] +) + +# Ordered dict to look for more specialized model mappings first +# before falling back to the more generic ones. +MODELS_TO_PIPELINE = OrderedDict( + [ + # Audio + ("MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES", "text-to-audio-spectrogram"), + ("MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES", "text-to-audio-waveform"), + ("MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "automatic-speech-recognition"), + ("MODEL_FOR_CTC_MAPPING_NAMES", "automatic-speech-recognition"), + ("MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES", "audio-frame-classification"), + ("MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "audio-classification"), + ("MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "audio-xvector"), + # Vision + ("MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES", "image-text-to-text"), + ("MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES", "image-to-text"), + ("MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES", "visual-question-answering"), + ("MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "depth-estimation"), + ("MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "video-classification"), + ("MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES", "zero-shot-image-classification"), + ("MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "image-classification"), + ("MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES", "zero-shot-object-detection"), + ("MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "object-detection"), + ("MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "image-segmentation"), + ("MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES", "image-to-image"), + ("MODEL_FOR_IMAGE_MAPPING_NAMES", "image-feature-extraction"), + # Text/tokens + ("MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "text-generation"), + ("MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES", "table-question-answering"), + ("MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES", "document-question-answering"), + ("MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "question-answering"), + ("MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "text2text-generation"), + ("MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES", "next-sentence-prediction"), + ("MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "multiple-choice"), + ("MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "text-classification"), + ("MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "token-classification"), + ("MODEL_FOR_MASKED_LM_MAPPING_NAMES", "fill-mask"), + ("MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "mask-generation"), + ("MODEL_FOR_PRETRAINING_MAPPING_NAMES", "pretraining"), + ] +) + + TF_TOKEN_CLASSIFICATION_SAMPLE = r""" Example: diff --git a/tests/utils/test_auto_docstring.py b/tests/utils/test_auto_docstring.py new file mode 100644 index 0000000000..11ac05f3b1 --- /dev/null +++ b/tests/utils/test_auto_docstring.py @@ -0,0 +1,88 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Team Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a clone of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + + +LLAMA_CLM_FORWARD = """ The [`LlamaForCausalLM`] forward method, overrides the `__call__` special method.\n\n \n\n Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`]\n instance afterwards instead of this since the former takes care of running the pre and post processing steps while\n the latter silently ignores them.\n\n \n\n Args:\n input_ids (`Optional[torch.LongTensor]`)of shape `(batch_size, sequence_length)`):\n Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.\n\n Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n [`PreTrainedTokenizer.__call__`] for details.\n\n [What are input IDs?](../glossary#input-ids)\n attention_mask (`Optional[torch.Tensor]`) of shape `(batch_size, sequence_length)`:\n Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:\n\n - 1 for tokens that are **not masked**,\n - 0 for tokens that are **masked**.\n\n [What are attention masks?](../glossary#attention-mask)\n position_ids (`Optional[torch.LongTensor]`):\n Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`.\n\n [What are position IDs?](../glossary#position-ids)\n past_key_values (`Optional[~cache_utils.Cache]`):\n Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention\n blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`\n returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.\n\n Two formats are allowed:\n - a `~cache_utils.Cache` instance, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);\n - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of\n shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy\n cache format.\n\n The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the\n legacy cache format will be returned.\n\n If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don\'t\n have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`\n of shape `(batch_size, sequence_length)`.\n inputs_embeds (`Optional[torch.FloatTensor]`):\n Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This\n is useful if you want more control over how to convert `input_ids` indices into associated vectors than the\n model\'s internal embedding lookup matrix.\n labels (`Optional[torch.LongTensor]`) of shape `(batch_size, sequence_length)`:\n Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,\n config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored\n (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.\n use_cache (`Optional[bool]`):\n If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see\n `past_key_values`).\n output_attentions (`Optional[bool]`):\n Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned\n tensors for more detail.\n output_hidden_states (`Optional[bool]`):\n Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n more detail.\n cache_position (`Optional[torch.LongTensor]`):\n Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,\n this tensor is not affected by padding. It is used to update the cache in the correct position and to infer\n the complete sequence length.\n logits_to_keep (`Union[int, torch.Tensor]`, defaults to `0`):\n If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all\n `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that\n token can save memory, which becomes pretty significant for long sequences or large vocabulary size.\n If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.\n This is useful when using packed tensor format (single dimension for batch and sequence length).\n\n Returns:\n [`transformers.modeling_outputs.CausalLMOutputWithPast`] or `tuple(torch.FloatTensor)`: A [`transformers.modeling_outputs.CausalLMOutputWithPast`] or a tuple of\n `torch.FloatTensor` (if `return_dict=False` is passed or when `config.return_dict=False`) comprising various\n elements depending on the configuration ([`LlamaConfig`]) and inputs.\n\n - **loss** (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided) -- Language modeling loss (for next-token prediction).\n - **logits** (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`) -- Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).\n - **past_key_values** (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`) -- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape\n `(batch_size, num_heads, sequence_length, embed_size_per_head)`)\n\n Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see\n `past_key_values` input) to speed up sequential decoding.\n - **hidden_states** (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`) -- Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +\n one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.\n\n Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.\n - **attentions** (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`) -- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,\n sequence_length)`.\n\n Attentions weights after the attention softmax, used to compute the weighted average in the self-attention\n heads.\n\n Example:\n\n ```python\n >>> from transformers import AutoTokenizer, LlamaForCausalLM\n\n >>> model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")\n >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")\n\n >>> prompt = "Hey, are you conscious? Can you talk to me?"\n >>> inputs = tokenizer(prompt, return_tensors="pt")\n\n >>> # Generate\n >>> generate_ids = model.generate(inputs.input_ids, max_length=30)\n >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]\n "Hey, are you conscious? Can you talk to me?\\nI\'m not conscious, but I can talk to you."\n ```""" + +LLAMA_MODEL_DOCSTRING = """ The [`LlamaModel`] forward method, overrides the `__call__` special method.\n\n \n\n Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`]\n instance afterwards instead of this since the former takes care of running the pre and post processing steps while\n the latter silently ignores them.\n\n \n\n Args:\n input_ids (`Optional[torch.LongTensor]`)of shape `(batch_size, sequence_length)`):\n Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.\n\n Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n [`PreTrainedTokenizer.__call__`] for details.\n\n [What are input IDs?](../glossary#input-ids)\n attention_mask (`Optional[torch.Tensor]`) of shape `(batch_size, sequence_length)`:\n Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:\n\n - 1 for tokens that are **not masked**,\n - 0 for tokens that are **masked**.\n\n [What are attention masks?](../glossary#attention-mask)\n position_ids (`Optional[torch.LongTensor]`):\n Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`.\n\n [What are position IDs?](../glossary#position-ids)\n past_key_values (`Optional[~cache_utils.Cache]`):\n Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention\n blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`\n returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.\n\n Two formats are allowed:\n - a `~cache_utils.Cache` instance, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);\n - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of\n shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy\n cache format.\n\n The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the\n legacy cache format will be returned.\n\n If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't\n have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`\n of shape `(batch_size, sequence_length)`.\n inputs_embeds (`Optional[torch.FloatTensor]`):\n Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This\n is useful if you want more control over how to convert `input_ids` indices into associated vectors than the\n model's internal embedding lookup matrix.\n use_cache (`Optional[bool]`):\n If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see\n `past_key_values`).\n output_attentions (`Optional[bool]`):\n Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned\n tensors for more detail.\n output_hidden_states (`Optional[bool]`):\n Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n more detail.\n cache_position (`Optional[torch.LongTensor]`):\n Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,\n this tensor is not affected by padding. It is used to update the cache in the correct position and to infer\n the complete sequence length.\n\n Returns:\n [`transformers.modeling_outputs.BaseModelOutputWithPast`] or `tuple(torch.FloatTensor)`: A [`transformers.modeling_outputs.BaseModelOutputWithPast`] or a tuple of\n `torch.FloatTensor` (if `return_dict=False` is passed or when `config.return_dict=False`) comprising various\n elements depending on the configuration ([`LlamaConfig`]) and inputs.\n\n - **last_hidden_state** (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) -- Sequence of hidden-states at the output of the last layer of the model.\n\n If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,\n hidden_size)` is output.\n - **past_key_values** (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`) -- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape\n `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if\n `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,\n encoder_sequence_length, embed_size_per_head)`.\n\n Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if\n `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`\n input) to speed up sequential decoding.\n - **hidden_states** (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`) -- Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +\n one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.\n\n Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.\n - **attentions** (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`) -- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,\n sequence_length)`.\n\n Attentions weights after the attention softmax, used to compute the weighted average in the self-attention\n heads.\n""" + +LLAMA_DECODER = """ The [`LlamaDecoderLayer`] forward method, overrides the `__call__` special method.\n\n \n\n Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`]\n instance afterwards instead of this since the former takes care of running the pre and post processing steps while\n the latter silently ignores them.\n\n \n\n Args:\n hidden_states (`torch.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim) attention_mask (`Optional[torch.Tensor]`) of shape `(batch_size, sequence_length)`:\n Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:\n\n - 1 for tokens that are **not masked**,\n - 0 for tokens that are **masked**.\n\n [What are attention masks?](../glossary#attention-mask)\n position_ids (`Optional[torch.LongTensor]`):\n Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`.\n\n [What are position IDs?](../glossary#position-ids)\n past_key_value (`Optional[~cache_utils.Cache]`):deprecated in favor of `past_key_values` output_attentions (`Optional[bool]`, defaults to `False`):\n Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned\n tensors for more detail.\n use_cache (`Optional[bool]`, defaults to `False`):\n If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see\n `past_key_values`).\n cache_position (`Optional[torch.LongTensor]`):\n Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,\n this tensor is not affected by padding. It is used to update the cache in the correct position and to infer\n the complete sequence length.\n position_embeddings (`Optional[Tuple[torch.Tensor, torch.Tensor]]`):\n Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,\n with `head_dim` being the embedding dimension of each attention head.\n\n Returns:\n `Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]`""" + +LLAMA_FOR_SEQUENCE_CLASSIFICATION_DOC = """ The [`LlamaForSequenceClassification`] forward method, overrides the `__call__` special method.\n\n \n\n Although the recipe for forward pass needs to be defined within this function, one should call the [`Module`]\n instance afterwards instead of this since the former takes care of running the pre and post processing steps while\n the latter silently ignores them.\n\n \n\n Args:\n input_ids (`Optional[torch.LongTensor]`)of shape `(batch_size, sequence_length)`):\n Indices of input sequence tokens in the vocabulary. Padding will be ignored by default.\n\n Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n [`PreTrainedTokenizer.__call__`] for details.\n\n [What are input IDs?](../glossary#input-ids)\n attention_mask (`Optional[torch.Tensor]`) of shape `(batch_size, sequence_length)`:\n Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:\n\n - 1 for tokens that are **not masked**,\n - 0 for tokens that are **masked**.\n\n [What are attention masks?](../glossary#attention-mask)\n position_ids (`Optional[torch.LongTensor]`):\n Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`.\n\n [What are position IDs?](../glossary#position-ids)\n past_key_values (`Optional[~cache_utils.Cache]`):\n Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention\n blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`\n returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.\n\n Two formats are allowed:\n - a `~cache_utils.Cache` instance, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);\n - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of\n shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy\n cache format.\n\n The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the\n legacy cache format will be returned.\n\n If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don\'t\n have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`\n of shape `(batch_size, sequence_length)`.\n inputs_embeds (`Optional[torch.FloatTensor]`):\n Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This\n is useful if you want more control over how to convert `input_ids` indices into associated vectors than the\n model\'s internal embedding lookup matrix.\n labels (`Optional[torch.LongTensor]`):\n Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,\n config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If\n `config.num_labels > 1` a classification loss is computed (Cross-Entropy).\n use_cache (`Optional[bool]`):\n If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see\n `past_key_values`).\n output_attentions (`Optional[bool]`):\n Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned\n tensors for more detail.\n output_hidden_states (`Optional[bool]`):\n Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n more detail.\n\n Returns:\n [`transformers.modeling_outputs.SequenceClassifierOutputWithPast`] or `tuple(torch.FloatTensor)`: A [`transformers.modeling_outputs.SequenceClassifierOutputWithPast`] or a tuple of\n `torch.FloatTensor` (if `return_dict=False` is passed or when `config.return_dict=False`) comprising various\n elements depending on the configuration ([`LlamaConfig`]) and inputs.\n\n - **loss** (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided) -- Classification (or regression if config.num_labels==1) loss.\n - **logits** (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`) -- Classification (or regression if config.num_labels==1) scores (before SoftMax).\n - **past_key_values** (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`) -- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape\n `(batch_size, num_heads, sequence_length, embed_size_per_head)`)\n\n Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see\n `past_key_values` input) to speed up sequential decoding.\n - **hidden_states** (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`) -- Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +\n one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.\n\n Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.\n - **attentions** (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`) -- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,\n sequence_length)`.\n\n Attentions weights after the attention softmax, used to compute the weighted average in the self-attention\n heads.\n\n Example of single-label classification:\n\n ```python\n >>> import torch\n >>> from transformers import AutoTokenizer, LlamaForSequenceClassification\n\n >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")\n >>> model = LlamaForSequenceClassification.from_pretrained("meta-llama/Llama-2-7b-hf")\n\n >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")\n\n >>> with torch.no_grad():\n ... logits = model(**inputs).logits\n\n >>> predicted_class_id = logits.argmax().item()\n >>> model.config.id2label[predicted_class_id]\n ...\n\n >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)`\n >>> num_labels = len(model.config.id2label)\n >>> model = LlamaForSequenceClassification.from_pretrained("meta-llama/Llama-2-7b-hf", num_labels=num_labels)\n\n >>> labels = torch.tensor([1])\n >>> loss = model(**inputs, labels=labels).loss\n >>> round(loss.item(), 2)\n ...\n ```\n\n Example of multi-label classification:\n\n ```python\n >>> import torch\n >>> from transformers import AutoTokenizer, LlamaForSequenceClassification\n\n >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")\n >>> model = LlamaForSequenceClassification.from_pretrained("meta-llama/Llama-2-7b-hf", problem_type="multi_label_classification")\n\n >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")\n\n >>> with torch.no_grad():\n ... logits = model(**inputs).logits\n\n >>> predicted_class_ids = torch.arange(0, logits.shape[-1])[torch.sigmoid(logits).squeeze(dim=0) > 0.5]\n\n >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)`\n >>> num_labels = len(model.config.id2label)\n >>> model = LlamaForSequenceClassification.from_pretrained(\n ... "meta-llama/Llama-2-7b-hf", num_labels=num_labels, problem_type="multi_label_classification"\n ... )\n\n >>> labels = torch.sum(\n ... torch.nn.functional.one_hot(predicted_class_ids[None, :].clone(), num_classes=num_labels), dim=1\n ... ).to(torch.float)\n >>> loss = model(**inputs, labels=labels).loss\n ```\n""" + +GEMMA3_IMAGE_PROCESSOR_FAST_DOCSTRING = """\nConstructs a fast Gemma3 image processor.\n\nParameters:\n do_resize (`Optional[bool]`, defaults to `True`):\n Whether to resize the image.\n size (`Optional[dict[str, int]]`, defaults to `{\'height\': 224, \'width\': 224}`):\n Describes the maximum input dimensions to the model.\n default_to_square (`Optional[bool]`, defaults to `True`):\n Whether to default to a square image when resizing, if size is an int.\n resample (`Union[PILImageResampling, F.InterpolationMode, NoneType]`, defaults to `2`):\n Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only\n has an effect if `do_resize` is set to `True`.\n do_center_crop (`Optional[bool]`, defaults to `None`):\n Whether to center crop the image.\n crop_size (`Optional[dict[str, int]]`, defaults to `None`):\n Size of the output image after applying `center_crop`.\n do_rescale (`Optional[bool]`, defaults to `True`):\n Whether to rescale the image.\n rescale_factor (`Union[int, float, NoneType]`, defaults to `0.00392156862745098`):\n Rescale factor to rescale the image by if `do_rescale` is set to `True`.\n do_normalize (`Optional[bool]`, defaults to `True`):\n Whether to normalize the image.\n image_mean (`Union[float, list[float], NoneType]`, defaults to `[0.5, 0.5, 0.5]`):\n Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.\n image_std (`Union[float, list[float], NoneType]`, defaults to `[0.5, 0.5, 0.5]`):\n Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to\n `True`.\n do_convert_rgb (`Optional[bool]`, defaults to `None`):\n Whether to convert the image to RGB.\n return_tensors (`Union[str, ~utils.generic.TensorType, NoneType]`, defaults to `None`):\n Returns stacked tensors if set to `pt, otherwise returns a list of tensors.\n data_format (`Optional[~image_utils.ChannelDimension]`, defaults to `ChannelDimension.FIRST`):\n Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors.\n input_data_format (`Union[str, ~image_utils.ChannelDimension, NoneType]`, defaults to `None`):\n The channel dimension format for the input image. If unset, the channel dimension format is inferred\n from the input image. Can be one of:\n - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.\n - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.\n - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.\n device (`Optional[torch.device]`, defaults to `None`):\n The device to process the images on. If unset, the device is inferred from the input images.\n do_pan_and_scan (`Optional[bool]`, defaults to `None`):\n Whether to apply `pan_and_scan` to images.\n pan_and_scan_min_crop_size (`Optional[int]`, defaults to `None`):\n Minimum size of each crop in pan and scan.\n pan_and_scan_max_num_crops (`Optional[int]`, defaults to `None`):\n Maximum number of crops per image in pan and scan.\n pan_and_scan_min_ratio_to_activate (`Optional[float]`, defaults to `None`):\n Minimum aspect ratio to activate pan and scan.\n""" + +GEMMA3_IMAGE_PROCESSOR_FAST_PREPROCESS_DOCSTRING = """ Args:\n images (`Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, list[\'PIL.Image.Image\'], list[numpy.ndarray], list[\'torch.Tensor\']]`):\n Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If\n passing in images with pixel values between 0 and 1, set `do_rescale=False`.\n do_resize (`Optional[bool]`):\n Whether to resize the image.\n size (`Optional[dict[str, int]]`):\n Describes the maximum input dimensions to the model.\n default_to_square (`Optional[bool]`):\n Whether to default to a square image when resizing, if size is an int.\n resample (`Union[PILImageResampling, F.InterpolationMode, NoneType]`):\n Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only\n has an effect if `do_resize` is set to `True`.\n do_center_crop (`Optional[bool]`):\n Whether to center crop the image.\n crop_size (`Optional[dict[str, int]]`):\n Size of the output image after applying `center_crop`.\n do_rescale (`Optional[bool]`):\n Whether to rescale the image.\n rescale_factor (`Union[int, float, NoneType]`):\n Rescale factor to rescale the image by if `do_rescale` is set to `True`.\n do_normalize (`Optional[bool]`):\n Whether to normalize the image.\n image_mean (`Union[float, list[float], NoneType]`):\n Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.\n image_std (`Union[float, list[float], NoneType]`):\n Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to\n `True`.\n do_convert_rgb (`Optional[bool]`):\n Whether to convert the image to RGB.\n return_tensors (`Union[str, ~utils.generic.TensorType, NoneType]`):\n Returns stacked tensors if set to `pt, otherwise returns a list of tensors.\n data_format (`Optional[~image_utils.ChannelDimension]`):\n Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors.\n input_data_format (`Union[str, ~image_utils.ChannelDimension, NoneType]`):\n The channel dimension format for the input image. If unset, the channel dimension format is inferred\n from the input image. Can be one of:\n - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.\n - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.\n - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.\n device (`Optional[torch.device]`):\n The device to process the images on. If unset, the device is inferred from the input images.\n do_pan_and_scan (`Optional[bool]`):\n Whether to apply `pan_and_scan` to images.\n pan_and_scan_min_crop_size (`Optional[int]`):\n Minimum size of each crop in pan and scan.\n pan_and_scan_max_num_crops (`Optional[int]`):\n Maximum number of crops per image in pan and scan.\n pan_and_scan_min_ratio_to_activate (`Optional[float]`):\n Minimum aspect ratio to activate pan and scan.\n\n Returns:\n ``:\n - **data** (`dict`) -- Dictionary of lists/arrays/tensors returned by the __call__ method (\'pixel_values\', etc.).\n - **tensor_type** (`Union[None, str, TensorType]`, *optional*) -- You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at\n initialization.\n""" + + +class AutoDocstringTest(unittest.TestCase): + pass + # def test_modeling_docstring(self): + # llama_docstring = " Args:\n images (`Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, list['PIL.Image.Image'], list[numpy.ndarray], list['torch.Tensor']]`):\n Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If\n passing in images with pixel values between 0 and 1, set `do_rescale=False`.\n do_resize (`Optional[bool]`):\n Whether to resize the image.\n size (`Optional[dict[str, int]]`):\n Describes the maximum input dimensions to the model.\n default_to_square (`Optional[bool]`):\n Whether to default to a square image when resizing, if size is an int.\n resample (`Union[PILImageResampling, F.InterpolationMode, NoneType]`):\n Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only\n has an effect if `do_resize` is set to `True`.\n do_center_crop (`Optional[bool]`):\n Whether to center crop the image.\n crop_size (`Optional[dict[str, int]]`):\n Size of the output image after applying `center_crop`.\n do_rescale (`Optional[bool]`):\n Whether to rescale the image.\n rescale_factor (`Union[int, float, NoneType]`):\n Rescale factor to rescale the image by if `do_rescale` is set to `True`.\n do_normalize (`Optional[bool]`):\n Whether to normalize the image.\n image_mean (`Union[float, list[float], NoneType]`):\n Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.\n image_std (`Union[float, list[float], NoneType]`):\n Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to\n `True`.\n do_convert_rgb (`Optional[bool]`):\n Whether to convert the image to RGB.\n return_tensors (`Union[str, ~utils.generic.TensorType, NoneType]`):\n Returns stacked tensors if set to `pt, otherwise returns a list of tensors.\n data_format (`Optional[~image_utils.ChannelDimension]`):\n Only `ChannelDimension.FIRST` is supported. Added for compatibility with slow processors.\n input_data_format (`Union[str, ~image_utils.ChannelDimension, NoneType]`):\n The channel dimension format for the input image. If unset, the channel dimension format is inferred\n from the input image. Can be one of:\n - `\"channels_first\"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.\n - `\"channels_last\"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.\n - `\"none\"` or `ChannelDimension.NONE`: image in (height, width) format.\n device (`Optional[torch.device]`):\n The device to process the images on. If unset, the device is inferred from the input images.\n do_pan_and_scan (`Optional[bool]`):\n Whether to apply `pan_and_scan` to images.\n pan_and_scan_min_crop_size (`Optional[int]`):\n Minimum size of each crop in pan and scan.\n pan_and_scan_max_num_crops (`Optional[int]`):\n Maximum number of crops per image in pan and scan.\n pan_and_scan_min_ratio_to_activate (`Optional[float]`):\n Minimum aspect ratio to activate pan and scan.\n\n Returns:\n ``:\n - **data** (`dict`) -- Dictionary of lists/arrays/tensors returned by the __call__ method ('pixel_values', etc.).\n - **tensor_type** (`Union[None, str, TensorType]`, *optional*) -- You can give a tensor_type here to convert the lists of integers in PyTorch/TensorFlow/Numpy Tensors at\n initialization.\n" + # self.assertEqual(llama_docstring, LlamaModel.__doc__) + + # self.assertEqual(LLAMA_MODEL_DOCSTRING, LlamaModel.forward.__doc__) + # self.assertEqual(LLAMA_CLM_FORWARD, LlamaForCausalLM.forward.__doc__) + # self.assertEqual(LLAMA_DECODER, LlamaDecoderLayer.forward.__doc__) + # self.assertEqual(LLAMA_FOR_SEQUENCE_CLASSIFICATION_DOC, LlamaForSequenceClassification.forward.__doc__) + + # def test_fast_image_processor_docstring(self): + # self.assertEqual(GEMMA3_IMAGE_PROCESSOR_FAST_DOCSTRING, Gemma3ImageProcessorFast.__doc__) + # self.assertEqual(GEMMA3_IMAGE_PROCESSOR_FAST_PREPROCESS_DOCSTRING, Gemma3ImageProcessorFast.preprocess.__doc__) + + # def test_auto_doc(self): + # COOL_CLASS_DOC = """ + # Args: + # input_ids (some): + # flash_attn_kwargs (FlashAttentionKwrargs): + # parameters that are completely optional and that should be passed. + # another_warg (something): should pass + # and_another_on (this time): + # I want + # this to be + # quite long + + # Example + + # ```python + # >>> import + # ``` + # """ + + # @auto_docstring + # class MyModel: + # @auto_docstring + # def __init__(input_ids, flash_attn_kwargs=None, another_warg=True, and_another_on=1): + # r""" + # Args: + # flash_attn_kwargs (FlashAttentionKwrargs): + # parameters that are completely optional and that should be passed. + # another_warg (something): should pass + # and_another_on (this time): + # I want + # this to be + # quite long + + # Example + + # ```python + # >>> import + # ``` + # """ + # pass + + # self.assertEqual(MyModel.__init__.__doc__, COOL_CLASS_DOC) diff --git a/utils/check_docstrings.py b/utils/check_docstrings.py index 154eda0cf1..00bc5a1220 100644 --- a/utils/check_docstrings.py +++ b/utils/check_docstrings.py @@ -36,8 +36,10 @@ like argument descriptions). import argparse import ast import enum +import glob import inspect import operator as op +import os import re from pathlib import Path from typing import Any, Optional, Tuple, Union @@ -46,6 +48,7 @@ from check_repo import ignore_undocumented from git import Repo from transformers.utils import direct_transformers_import +from transformers.utils.args_doc import ImageProcessorArgs, ModelArgs, parse_docstring, set_min_indent, source_args_doc PATH_TO_REPO = Path(__file__).parent.parent.resolve() @@ -959,6 +962,404 @@ def fix_docstring(obj: Any, old_doc_args: str, new_doc_args: str): f.write("\n".join(lines)) +def _find_sig_line(lines, line_end): + parenthesis_count = 0 + sig_line_end = line_end + found_sig = False + while not found_sig: + for char in lines[sig_line_end]: + if char == "(": + parenthesis_count += 1 + elif char == ")": + parenthesis_count -= 1 + if parenthesis_count == 0: + found_sig = True + break + sig_line_end += 1 + return sig_line_end + + +def find_matching_model_files(check_all: bool = False): + """ + Find all model files in the transformers repo that should be checked for @auto_docstring, + excluding files with certain substrings. + Returns: + List of file paths. + """ + module_diff_files = None + if not check_all: + module_diff_files = set() + repo = Repo(PATH_TO_REPO) + # Diff from index to unstaged files + for modified_file_diff in repo.index.diff(None): + if modified_file_diff.a_path.startswith("src/transformers"): + module_diff_files.add(os.path.join(PATH_TO_REPO, modified_file_diff.a_path)) + # Diff from index to `main` + for modified_file_diff in repo.index.diff(repo.refs.main.commit): + if modified_file_diff.a_path.startswith("src/transformers"): + module_diff_files.add(os.path.join(PATH_TO_REPO, modified_file_diff.a_path)) + # quick escape route: if there are no module files in the diff, skip this check + if len(module_diff_files) == 0: + return None + + modeling_glob_pattern = os.path.join(PATH_TO_TRANSFORMERS, "models/**/modeling_**") + potential_files = glob.glob(modeling_glob_pattern) + image_processing_glob_pattern = os.path.join(PATH_TO_TRANSFORMERS, "models/**/image_processing_*_fast.py") + potential_files += glob.glob(image_processing_glob_pattern) + exclude_substrings = ["modeling_tf_", "modeling_flax_"] + matching_files = [] + for file_path in potential_files: + if os.path.isfile(file_path): + filename = os.path.basename(file_path) + is_excluded = any(exclude in filename for exclude in exclude_substrings) + if not is_excluded: + matching_files.append(file_path) + if not check_all: + # intersect with module_diff_files + matching_files = sorted([file for file in matching_files if file in module_diff_files]) + + print(" Checking auto_docstrings in the following files:" + "\n - " + "\n - ".join(matching_files)) + + return matching_files + + +def find_files_with_auto_docstring(matching_files, decorator="@auto_docstring"): + """ + From a list of files, return those that contain the @auto_docstring decorator. + """ + auto_docstrings_files = [] + for file_path in matching_files: + with open(file_path, "r", encoding="utf-8") as f: + content_base_file = f.read() + if decorator in content_base_file: + lines = content_base_file.split("\n") + line_numbers = [i for i, line in enumerate(lines) if decorator in line] + for line_number in line_numbers: + line_end = line_number + end_patterns = ["class ", " def"] + stop_condition = False + while line_end < len(lines) and not stop_condition: + line_end += 1 + stop_condition = any(lines[line_end].startswith(end_pattern) for end_pattern in end_patterns) + candidate_patterns = ["class ", " def"] + candidate = any( + lines[line_end].startswith(candidate_pattern) for candidate_pattern in candidate_patterns + ) + if stop_condition and candidate: + auto_docstrings_files.append(file_path) + break + return auto_docstrings_files + + +def get_auto_docstring_candidate_lines(lines): + """ + For a file's lines, find the start and end line indices of all @auto_docstring candidates. + Returns two lists: starts and ends. + """ + line_numbers = [i for i, line in enumerate(lines) if "@auto_docstring" in line] + line_starts_candidates = [] + line_ends_candidates = [] + for line_number in line_numbers: + line_end = line_number + end_patterns = ["class ", " def"] + stop_condition = False + while line_end < len(lines) and not stop_condition: + line_end += 1 + stop_condition = any(lines[line_end].startswith(end_pattern) for end_pattern in end_patterns) + candidate_patterns = ["class ", " def"] + candidate = any(lines[line_end].startswith(candidate_pattern) for candidate_pattern in candidate_patterns) + if stop_condition and candidate: + line_ends_candidates.append(line_end) + line_starts_candidates.append(line_number) + return line_starts_candidates, line_ends_candidates + + +def generate_new_docstring_for_signature( + lines, + sig_start_line, + sig_end_line, + docstring_line, + arg_indent=" ", + custom_args_dict={}, +): + """ + Generalized docstring generator for a function or class signature. + Args: + lines: List of lines from the file. + sig_start_line: Line index where the signature starts. + sig_end_line: Line index where the signature ends. + docstring_line: Line index where the docstring starts (or None if not present). + arg_indent: Indentation for missing argument doc entries. + Returns: + new_docstring, sig_end_line, docstring_end (last docstring line index) + """ + # Extract and clean signature + missing_docstring_args = [] + fill_docstring_args = [] + + signature_content = lines[sig_start_line:sig_end_line] + signature_content = [line.split("#")[0] for line in signature_content] + signature_content = "".join(signature_content) + signature_content = "".join(signature_content.split(")")[:-1]) + args_in_signature = re.findall(r"[,(]\s*(\w+)\s*(?=:|=|,|\))", signature_content) + if "self" in args_in_signature: + args_in_signature.remove("self") + # Parse docstring if present + args_docstring_dict = {} + remaining_docstring = "" + docstring_end = sig_end_line - 1 + if docstring_line is not None: + docstring_end = docstring_line + if not lines[docstring_line].count('"""') >= 2: + docstring_end += 1 + while '"""' not in lines[docstring_end]: + docstring_end += 1 + docstring_content = lines[docstring_line : docstring_end + 1] + parsed_docstring, remaining_docstring = parse_docstring("\n".join(docstring_content)) + args_docstring_dict.update(parsed_docstring) + # Fill missing args + for arg in args_in_signature: + if ( + arg not in args_docstring_dict + and arg not in source_args_doc([ModelArgs, ImageProcessorArgs]) + and arg not in custom_args_dict + ): + missing_docstring_args.append(arg) + args_docstring_dict[arg] = { + "type": "", + "optional": False, + "shape": None, + "description": f"\n{arg_indent} ", + "default": None, + "additional_info": None, + } + # Build new docstring + new_docstring = "" + if len(args_docstring_dict) > 0 or remaining_docstring: + new_docstring += 'r"""\n' + for arg in args_docstring_dict: + additional_info = args_docstring_dict[arg]["additional_info"] or "" + custom_arg_description = args_docstring_dict[arg]["description"] + if "" in custom_arg_description and arg not in missing_docstring_args: + fill_docstring_args.append(arg) + if custom_arg_description.endswith('"""'): + custom_arg_description = "\n".join(custom_arg_description.split("\n")[:-1]) + new_docstring += f"{arg} ({args_docstring_dict[arg]['type']}{additional_info}):{custom_arg_description}\n" + close_docstring = True + if remaining_docstring: + if remaining_docstring.endswith('"""'): + close_docstring = False + end_docstring = "\n" if close_docstring else "" + new_docstring += f"{set_min_indent(remaining_docstring, 0)}{end_docstring}" + if close_docstring: + new_docstring += '"""' + new_docstring = set_min_indent(new_docstring, 8) + return new_docstring, sig_end_line, docstring_end, missing_docstring_args, fill_docstring_args + + +def generate_new_docstring_for_function(lines, current_line_end, custom_args_dict): + """ + Wrapper for function docstring generation using the generalized helper. + """ + sig_line_end = _find_sig_line(lines, current_line_end) + docstring_line = sig_line_end if '"""' in lines[sig_line_end] else None + return generate_new_docstring_for_signature( + lines, + current_line_end, + sig_line_end, + docstring_line, + arg_indent=" ", + custom_args_dict=custom_args_dict, + ) + + +def generate_new_docstring_for_class(lines, current_line_end, custom_args_dict): + """ + Wrapper for class docstring generation (via __init__) using the generalized helper. + Returns the new docstring and relevant signature/docstring indices. + """ + init_method_line = current_line_end + found_init_method = False + while init_method_line < len(lines) - 1 and not found_init_method: + init_method_line += 1 + if " def __init__" in lines[init_method_line]: + found_init_method = True + elif lines[init_method_line].startswith("class "): + break + if not found_init_method: + return "", None, None, None, [], [] + init_method_sig_line_end = _find_sig_line(lines, init_method_line) + docstring_line = init_method_sig_line_end if '"""' in lines[init_method_sig_line_end] else None + new_docstring, _, init_method_docstring_end, missing_docstring_args, fill_docstring_args = ( + generate_new_docstring_for_signature( + lines, + init_method_line, + init_method_sig_line_end, + docstring_line, + arg_indent="", + custom_args_dict=custom_args_dict, + ) + ) + return ( + new_docstring, + init_method_line, + init_method_sig_line_end, + init_method_docstring_end, + missing_docstring_args, + fill_docstring_args, + ) + + +def find_custom_args_with_details(file_content: str, custom_args_var_name: str) -> list[dict]: + """ + Find the given custom args variable in the file content and return its content. + + Args: + file_content: The string content of the Python file. + custom_args_var_name: The name of the custom args variable. + """ + # Escape the variable_name to handle any special regex characters it might contain + escaped_variable_name = re.escape(custom_args_var_name) + + # Construct the regex pattern dynamically with the specific variable name + # This regex looks for: + # ^\s* : Start of a line with optional leading whitespace. + # ({escaped_variable_name}) : Capture the exact variable name. + # \s*=\s* : An equals sign, surrounded by optional whitespace. + # (r?\"\"\") : Capture the opening triple quotes (raw or normal string). + # (.*?) : Capture the content (non-greedy). + # (\"\"\") : Match the closing triple quotes. + regex_pattern = rf"^\s*({escaped_variable_name})\s*=\s*(r?\"\"\")(.*?)(\"\"\")" + + flags = re.MULTILINE | re.DOTALL + + # Use re.search to find the first match + match = re.search(regex_pattern, file_content, flags) + + if match: + # match.group(1) will be the variable_name itself + # match.group(3) will be the content inside the triple quotes + content = match.group(3).strip() + return content + return None + + +def update_file_with_new_docstrings( + candidate_file, lines, line_starts_candidates, line_ends_candidates, overwrite=False +): + """ + For a given file, update the docstrings for all @auto_docstring candidates and write the new content. + """ + content_base_file_new_lines = lines[: line_ends_candidates[0]] + current_line_start = line_starts_candidates[0] + current_line_end = line_ends_candidates[0] + index = 1 + missing_docstring_args_warnings = [] + + fill_docstring_args_warnings = [] + while index <= len(line_starts_candidates): + custom_args_dict = {} + auto_docstring_signature_content = "".join(lines[current_line_start:current_line_end]) + match = re.findall(r"custom_args=(\w+)", auto_docstring_signature_content) + if match: + custom_args_var_name = match[0] + custom_args_var_content = find_custom_args_with_details("\n".join(lines), custom_args_var_name) + if custom_args_var_content: + custom_args_dict, _ = parse_docstring(custom_args_var_content) + new_docstring = "" + found_init_method = False + # Function + if " def" in lines[current_line_end]: + new_docstring, sig_line_end, docstring_end, missing_docstring_args, fill_docstring_args = ( + generate_new_docstring_for_function(lines, current_line_end, custom_args_dict) + ) + # Class + elif "class " in lines[current_line_end]: + ( + new_docstring, + init_method_line, + init_method_sig_line_end, + init_method_docstring_end, + missing_docstring_args, + fill_docstring_args, + ) = generate_new_docstring_for_class(lines, current_line_end, custom_args_dict) + found_init_method = init_method_line is not None + # Add warnings if needed + if missing_docstring_args: + for arg in missing_docstring_args: + missing_docstring_args_warnings.append(f" - {arg} line {current_line_end}") + if fill_docstring_args: + for arg in fill_docstring_args: + fill_docstring_args_warnings.append(f" - {arg} line {current_line_end}") + + # Write new lines + if index >= len(line_ends_candidates) or line_ends_candidates[index] > current_line_end: + if " def" in lines[current_line_end]: + content_base_file_new_lines += lines[current_line_end:sig_line_end] + if new_docstring != "": + content_base_file_new_lines += new_docstring.split("\n") + if index < len(line_ends_candidates): + content_base_file_new_lines += lines[docstring_end + 1 : line_ends_candidates[index]] + else: + content_base_file_new_lines += lines[docstring_end + 1 :] + elif found_init_method: + content_base_file_new_lines += lines[current_line_end:init_method_sig_line_end] + if new_docstring != "": + content_base_file_new_lines += new_docstring.split("\n") + if index < len(line_ends_candidates): + content_base_file_new_lines += lines[init_method_docstring_end + 1 : line_ends_candidates[index]] + else: + content_base_file_new_lines += lines[init_method_docstring_end + 1 :] + elif index < len(line_ends_candidates): + content_base_file_new_lines += lines[current_line_end : line_ends_candidates[index]] + else: + content_base_file_new_lines += lines[current_line_end:] + if index < len(line_ends_candidates): + current_line_end = line_ends_candidates[index] + current_line_start = line_starts_candidates[index] + index += 1 + content_base_file_new = "\n".join(content_base_file_new_lines) + if overwrite: + with open(candidate_file, "w", encoding="utf-8") as f: + f.write(content_base_file_new) + + return missing_docstring_args_warnings, fill_docstring_args_warnings + + +def check_auto_docstrings(overwrite: bool = False, check_all: bool = False): + """ + Check docstrings of all public objects that are decorated with `@auto_docstrings`. + This function orchestrates the process by finding relevant files, scanning for decorators, + generating new docstrings, and updating files as needed. + """ + # 1. Find all model files to check + matching_files = find_matching_model_files(check_all) + if matching_files is None: + return + # 2. Find files that contain the @auto_docstring decorator + auto_docstrings_files = find_files_with_auto_docstring(matching_files) + # 3. For each file, update docstrings for all candidates + for candidate_file in auto_docstrings_files: + with open(candidate_file, "r", encoding="utf-8") as f: + lines = f.read().split("\n") + line_starts_candidates, line_ends_candidates = get_auto_docstring_candidate_lines(lines) + missing_docstring_args_warnings, fill_docstring_args_warnings = update_file_with_new_docstrings( + candidate_file, lines, line_starts_candidates, line_ends_candidates, overwrite=overwrite + ) + if missing_docstring_args_warnings: + if not overwrite: + print( + "Some docstrings are missing. Run `make fix-copies` or `python utils/check_docstrings.py --fix_and_overwrite` to generate the docstring templates where needed." + ) + print(f"🚨 Missing docstring for the following arguments in {candidate_file}:") + for warning in missing_docstring_args_warnings: + print(warning) + if fill_docstring_args_warnings: + print(f"🚨 Docstring needs to be filled for the following arguments in {candidate_file}:") + for warning in fill_docstring_args_warnings: + print(warning) + + def check_docstrings(overwrite: bool = False, check_all: bool = False): """ Check docstrings of all public objects that are callables and are documented. By default, only checks the diff. @@ -1017,6 +1418,9 @@ def check_docstrings(overwrite: bool = False, check_all: bool = False): hard_failures.append(name) continue if old_doc != new_doc: + print("name", name) + print("old_doc", old_doc) + print("new_doc", new_doc) if overwrite: fix_docstring(obj, old_doc, new_doc) else: @@ -1059,5 +1463,5 @@ if __name__ == "__main__": "--check_all", action="store_true", help="Whether to check all files. By default, only checks the diff" ) args = parser.parse_args() - + check_auto_docstrings(overwrite=args.fix_and_overwrite, check_all=args.check_all) check_docstrings(overwrite=args.fix_and_overwrite, check_all=args.check_all) diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index 2a383123ba..9a25b3ec04 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -258,7 +258,7 @@ def get_docstring_indent(docstring): return 0 -def is_full_docstring(new_docstring: str) -> bool: +def is_full_docstring(original_docstring: str, new_docstring: str, original_level: int) -> bool: """Check if `new_docstring` is a full docstring, or if it is only part of a docstring that should then be merged with the existing old one. """ @@ -267,6 +267,17 @@ def is_full_docstring(new_docstring: str) -> bool: # The docstring contains Args definition, so it is self-contained if re.search(r"\n\s*Args:\n", new_docstring): return True + elif re.search(r"\n\s*Args:\n", original_docstring): + return False + # Check if the docstring contains args docstring (meaning it is self contained): + param_pattern = re.compile( + # |--- Group 1 ---|| Group 2 ||- Group 3 -||---------- Group 4 ----------| + rf"^\s{{0,{original_level}}}(\w+)\s*\(\s*([^, \)]*)(\s*.*?)\s*\)\s*:\s*((?:(?!\n^\s{{0,{original_level}}}\w+\s*\().)*)", + re.DOTALL | re.MULTILINE, + ) + match_object = param_pattern.search(new_docstring) + if match_object is not None: + return True # If it contains Returns, but starts with text indented with an additional 4 spaces before, it is self-contained # (this is the scenario when using `@add_start_docstrings_to_model_forward`, but adding more args to docstring) match_object = re.search(r"\n([^\S\n]*)Returns:\n", new_docstring) @@ -280,7 +291,7 @@ def is_full_docstring(new_docstring: str) -> bool: def merge_docstrings(original_docstring, updated_docstring): original_level = get_docstring_indent(original_docstring) - if not is_full_docstring(updated_docstring): + if not is_full_docstring(original_docstring, updated_docstring, original_level): # Split the docstring at the example section, assuming `"""` is used to define the docstring parts = original_docstring.split("```") if "```" in updated_docstring and len(parts) > 1: @@ -291,13 +302,22 @@ def merge_docstrings(original_docstring, updated_docstring): parts[1] = new_parts[1] updated_docstring = "".join( [ - parts[0].rstrip(" \n") + new_parts[0], f"\n{original_level * ' '}```", parts[1], "```", parts[2], ] ) + docstring_opening, original_start_docstring = parts[0].rstrip(" \n").split('"""')[:2] + new_start_docstring = new_parts[0].rstrip(" \n") + docstring_opening += '"""' + if new_start_docstring.startswith(original_start_docstring): + updated_docstring = new_start_docstring + "\n" + updated_docstring + elif original_start_docstring.endswith(new_start_docstring): + updated_docstring = original_start_docstring + "\n" + updated_docstring + else: + updated_docstring = original_start_docstring + "\n" + new_start_docstring + "\n" + updated_docstring + updated_docstring = docstring_opening + updated_docstring elif updated_docstring not in original_docstring: # add tabulation if we are at the lowest level. if re.search(r"\n\s*.*\(.*\)\:\n\s*\w", updated_docstring):